diff --git a/bin/prclc b/bin/prclc deleted file mode 100644 index eba45cd..0000000 --- a/bin/prclc +++ /dev/null @@ -1,24 +0,0 @@ -import argparse -import os -from os import path -from sys import stderr - -from tbxi.prclc import compile - - -parser = argparse.ArgumentParser(description=''' - Parcel blob compiler -''') - -parser.add_argument('source', nargs='?', default=os.getcwd(), help='Parcelfile or directory') -parser.add_argument('-o', metavar='dest-file', default='MacOSROM', help='output file (default: MacOSROM)') - -args = parser.parse_args() - -if path.isdir(args.source): - args.source = path.join(args.source, 'Parcelfile') - -result = compile(args.source) - -with open(args.o, 'wb') as f: - f.write(result) diff --git a/bin/prcldump b/bin/prcldump deleted file mode 100644 index 9ee73c4..0000000 --- a/bin/prcldump +++ /dev/null @@ -1,53 +0,0 @@ -import argparse -import os -from os import path -from sys import stderr - -from tbxi.lowlevel import MAGIC -from tbxi.prcldump import dump - - -parser = argparse.ArgumentParser(description=''' - Dump a MacOS parcel blob (magic number 0x7072636C 'prcl') to a - plain-text Parcelfile and several decompressed binaries. This output - can be rebuilt using the Parcel Compiler (prclc). Usually parcel - blobs are found embedded inside a file called "Mac OS ROM", although - the Blue Box uses them in isolation. As a convenience this utility - will search for the magic number inside any input file (with a - warning). -''') - -parser.add_argument('source', nargs=1, help='file to be decompiled') - -meg = parser.add_mutually_exclusive_group() -meg.add_argument('-d', metavar='dest-dir', help='output directory (Parcelfile will be created within)') -meg.add_argument('-f', metavar='dest-file', help='output file (binaries will go in parent directory)') - -args = parser.parse_args() - -with open(args.source[0], 'rb') as f: - binary = f.read() - -if not binary.startswith(MAGIC): - try: - offset = binary.index(MAGIC) - except ValueError: - print('Not a parcels file', file=stderr) - exit(1) - else: - print('Warning: parcel blob wrapped at offset 0x%x' % offset) - binary = binary[offset:] - -if args.f: - dest_file = path.abspath(args.f) - dest_dir = path.dirname(dest_file) -elif args.d: - dest_dir = path.abspath(args.d) - dest_file = path.join(dest_dir, 'Parcelfile') -else: - dest_dir = path.abspath(args.source[0].rstrip(path.sep) + '-dump') - dest_file = path.join(dest_dir, 'Parcelfile') - -os.makedirs(dest_dir, exist_ok=True) - -dump(binary, dest_file, dest_dir) diff --git a/setup.py b/setup.py index 6f00e80..6bbaa1f 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,9 @@ setup_args = dict( 'Topic :: Software Development :: Build Tools', 'License :: OSI Approved :: MIT License', ], + zip_safe=True, packages=['tbxi'], - scripts=['bin/prclc', 'bin/prcldump'], + entry_points=dict(console_scripts=['tbxi = tbxi.__main__:main']), ext_modules=[Extension('tbxi.fast_lzss', ['speedups/fast_lzss.c'])], ) @@ -27,5 +28,6 @@ setup_args = dict( try: setup(**setup_args) except (SystemExit, Exception): + raise setup_args.pop('ext_modules') setup(**setup_args) diff --git a/tbxi/__main__.py b/tbxi/__main__.py new file mode 100644 index 0000000..52076e3 --- /dev/null +++ b/tbxi/__main__.py @@ -0,0 +1,68 @@ +# Thanks to Chris Warrick for script installation tips +# https://chriswarrick.com/blog/2014/09/15/python-apps-the-right-way-entry_points-and-scripts/ + +import argparse +import sys +import os +from os import path +import shutil + +from .slow_lzss import decompress + +from . import dispatcher + + +def main(args=None): + if args is None: args = sys.argv[1:] + + descriptions = { + 'dump': '''Break a ROM file into rebuildable parts. Any OldWorld + or NewWorld image can be processed. Because successive ROM + formats tended to wrap layers around old ones, up to four layers + can require 'unpeeling'.''', + 'build': '''Recreate a dumped ROM file. With minor exceptions, + the result should be identical to the original.''' + } + + for key in list(descriptions): + descriptions[key] = ' '.join(descriptions[key].split()) + + if not args or args[0] not in descriptions: + print('usage: tbxi [...]') + print() + print('The Mac OS Toolbox Imager') + print() + print('commands:') + for k, v in descriptions.items(): + print(' ' + k.ljust(8) + ' ' + v.partition('.')[0]) + exit(1) + + command = args.pop(0) + parser = argparse.ArgumentParser(prog='tbxi ' + command, description=descriptions[command]) + + if command == 'dump': + parser.add_argument('file', metavar='', help='original file (dest: .src') + parser.add_argument('-o', dest='output', metavar='', help='destination (default: .src)') + args = parser.parse_args(args) + + if not args.output: args.output = args.file + '.src' + + with open(args.file, 'rb') as f: + try: + shutil.rmtree(args.output) + except FileNotFoundError: + pass + + dispatcher.dump(f.read(), args.output, toplevel=True) + + elif command == 'build': + parser.add_argument('dir', metavar='', help='source directory') + parser.add_argument('-o', dest='output', metavar='', help='destination (default: .build)') + args = parser.parse_args(args) + + with open(args.output, 'wb') as f: + f.write(dispatcher.build_path(args.dir)) + + +if __name__ == "__main__": + main() diff --git a/tbxi/binescape.py b/tbxi/binescape.py new file mode 100644 index 0000000..145a30c --- /dev/null +++ b/tbxi/binescape.py @@ -0,0 +1,9 @@ +from ast import literal_eval + +def ascii_from_bytes(s): + # convert to b'stuff' form, then cut off the first two and last one chars + return ''.join(repr(bytes([c]))[2:-1] for c in s) + +def bytes_from_ascii(s): + # undo the above + return b'"'.join(literal_eval('b"' + part + '"') for part in s.split('"')).decode('ascii') diff --git a/tbxi/bootinfo_dump.py b/tbxi/bootinfo_dump.py new file mode 100644 index 0000000..0bbdd67 --- /dev/null +++ b/tbxi/bootinfo_dump.py @@ -0,0 +1,47 @@ +import os +from os import path +import re + +from . import dispatcher + + +def dump(binary, dest_dir): + if not binary.startswith(b''): raise dispatcher.WrongFormat + + os.makedirs(dest_dir, exist_ok=True) + + a, b, c = binary.partition(b'') + chrp_boot = a + b + if c.startswith(b'\r'): chrp_boot += b'\r' + + chrp_boot = chrp_boot.replace(b'\r', b'\n') + + # find the build-specific hex, and write out a clean version of the script + chrp_boot_zeroed = bytearray(chrp_boot) + constants = dict() + for m in re.finditer(rb'h#\s+([A-Fa-f0-9]+)\s+constant\s+([-\w]+)', chrp_boot): + key = m.group(2).decode('ascii') + val = int(m.group(1), 16) + constants[key] = val + + for i in range(*m.span(1)): + chrp_boot_zeroed[i:i+1] = b'0' + + with open(path.join(dest_dir, 'Bootscript'), 'wb') as f: + f.write(chrp_boot_zeroed) + + if 'elf-offset' in constants: + elf = binary[constants['elf-offset']:][:constants['elf-size']] + dispatcher.dump(elf, path.join(dest_dir, 'MacOS.elf')) + + other_offset = constants.get('lzss-offset', constants.get('parcels-offset')) + other_size = constants.get('lzss-size', constants.get('parcels-size')) + parcels = binary[other_offset:][:other_size] + + if parcels.startswith(b'prcl'): + filename = 'Parcels' + else: + filename = 'MacROM' + parcels = decompress(parcels) + + dispatcher.dump(parcels, path.join(dest_dir, filename)) diff --git a/tbxi/dispatcher.py b/tbxi/dispatcher.py new file mode 100644 index 0000000..0e76d1b --- /dev/null +++ b/tbxi/dispatcher.py @@ -0,0 +1,78 @@ +import importlib + +from subprocess import run, PIPE + +import os +from os import path + + +FORMATS = ''' + bootinfo + parcels + powerpc + supermario +'''.split() + + +class WrongFormat(Exception): + pass + + +def build_dir(p): + for fmt in FORMATS: + mod = importlib.import_module('..%s_build' % fmt, __name__) + try: + data = mod.build(p) + except WrongFormat: + continue + + print(fmt) + return data + + raise WrongFormat + + +def build_path(p): + parent = path.dirname(path.abspath(p)) + name = path.basename(path.abspath(p)) + + # try building the file from a directory + for np in [p + '.src', p]: + try: + data = build_dir(p) + except WrongFormat: + pass + else: + break + else: + # fall back on just reading the file (boring, I know!) + with open(p, 'rb') as f: + data = f.read() + + # Search the directory of the file for executable patches + for sib in sorted(os.scandir(parent), key=lambda ent: ent.name): + # Does the filename match? + if sib.name.startswith(name) and 'patch' in sib.name[len(name):]: + # This is a bit unsafe, so prompt the user (to pipe in `yes`...) + if input('Apply %s to %s? [y/N]' % (sib.name, name)).lower().startswith('y'): + result = run([sib.path], cwd=parent, stdin=data, stdout=PIPE, check=True) + data = result.stdout + + return data + + +def dump(binary, dest_path, toplevel=False): + if not toplevel: + with open(dest_path, 'wb') as f: + f.write(binary) + + dest_path += '.src' + + for fmt in FORMATS: + mod = importlib.import_module('..%s_dump' % fmt, __name__) + try: + mod.dump(binary, dest_path) + did_dump = True + break + except WrongFormat: + pass diff --git a/tbxi/lowlevel.py b/tbxi/lowlevel.py index 1ac329e..0995ec8 100644 --- a/tbxi/lowlevel.py +++ b/tbxi/lowlevel.py @@ -6,8 +6,67 @@ class MyParcelStruct(NamedTupleStruct, StringStruct): MAGIC = b'prcl\x01\x00\x00\x00' +def pad(offset, align): + x = offset + align - 1 + x -= x % align + return x + PrclNodeStruct = MyParcelStruct('>I 4s I I I I 32s 32s', name='PrclNodeStruct', fields=['link', 'ostype', 'hdr_size', 'flags', 'n_children', 'child_size', 'a', 'b']) PrclChildStruct = MyParcelStruct('>4s I 4s I I I I 32s', name='PrclChildStruct', fields=['ostype', 'flags', 'compress', 'unpackedlen', 'cksum', 'packedlen', 'ptr', 'name']) + +SuperMarioHeader = NamedTupleStruct('>I I B B I I H B B L L L L L L B B 4L L L L L', name='SuperMarioHeader', + fields=['CheckSum', 'ResetPC', 'MachineNumber', 'ROMVersion', + 'ReStartJMP', 'BadDiskJMP', 'ROMRelease', 'PatchFlags', 'unused1', + 'ForeignOSTbl', 'RomRsrc', 'EjectJMP', 'DispTableOff', + 'CriticalJMP', 'ResetEntryJMP', 'RomLoc', 'unused2', 'CheckSum0', + 'CheckSum1', 'CheckSum2', 'CheckSum3', 'RomSize', 'EraseIconOff', + 'InitSys7ToolboxOff', 'SubVers']) + +SuperMarioForeignOS = NamedTupleStruct('>7I', name='SuperMarioForeignOS', + fields=['InitDispatcher', 'EMT1010', 'BadTrap', 'StartSDeclMgr', + 'InitMemVect', 'SwitchMMU', 'InitRomVectors']) + +COMBO_FIELDS = { + 0x40 << 56: 'AppleTalk1', + 0x20 << 56: 'AppleTalk2', + 0x30 << 56: 'AppleTalk2_NetBoot_FPU', + 0x08 << 56: 'AppleTalk2_NetBoot_NoFPU', + 0x10 << 56: 'NetBoot', + 0x78 << 56: 'AllCombos', +} + +ResHeader = NamedTupleStruct('>L B B H H 6x', name='ResHeader', + fields=['offsetToFirst', 'maxValidIndex', 'comboFieldSize', + 'comboVersion', 'headerSize']) + +ResEntry = NamedTupleStruct('>Q L L 4s h B 256p', name='ResEntry', + fields=['combo', 'offsetToNext', 'offsetToData', 'rsrcType', + 'rsrcID', 'rsrcAttr', 'rsrcName']) + +def ResEntry_padded_len(entry_tuple): + return pad(ResEntry.size - 256 + len(entry_tuple.rsrcName), 16) + +FakeMMHeader = NamedTupleStruct('>4s L L L', name='FakeMMHeader', + fields=['MagicKurt', 'MagicC0A00000', 'dataSizePlus12', 'bogusOff']) + +ConfigInfo = NamedTupleStruct('>40x lLL lL lL lL lL lL lL 16sLLL LL LLLLbxxx LLLLL L LLLLLL 128s128s128s128s 128s LLLL L L lL LLL', name='ConfigInfo', + fields=['ROMImageBaseOffset', 'ROMImageSize', 'ROMImageVersion', + 'Mac68KROMOffset', 'Mac68KROMSize', 'ExceptionTableOffset', + 'ExceptionTableSize', 'HWInitCodeOffset', 'HWInitCodeSize', + 'KernelCodeOffset', 'KernelCodeSize', 'EmulatorCodeOffset', + 'EmulatorCodeSize', 'OpcodeTableOffset', 'OpcodeTableSize', + 'BootstrapVersion', 'BootVersionOffset', 'ECBOffset', + 'IplValueOffset', 'EmulatorEntryOffset', 'KernelTrapTableOffset', + 'TestIntMaskInit', 'ClearIntMaskInit', 'PostIntMaskInit', + 'LA_InterruptCtl', 'InterruptHandlerKind', 'LA_InfoRecord', + 'LA_KernelData', 'LA_EmulatorData', 'LA_DispatchTable', + 'LA_EmulatorCode', 'MacLowMemInitOffset', 'PageAttributeInit', + 'PageMapInitSize', 'PageMapInitOffset', 'PageMapIRPOffset', + 'PageMapKDPOffset', 'PageMapEDPOffset', 'SegMap32SupInit', 'SegMap32UsrInit', 'SegMap32CPUInit', 'SegMap32OvlInit', 'BATRangeInit', + 'BatMap32SupInit', 'BatMap32UsrInit', 'BatMap32CPUInit', + 'BatMap32OvlInit', 'SharedMemoryAddr', 'PA_RelocatedLowMemInit', + 'OpenFWBundleOffset', 'OpenFWBundleSize', 'LA_OpenFirmware', + 'PA_OpenFirmware', 'LA_HardwarePriv']) diff --git a/tbxi/prclc.py b/tbxi/parcels_build.py similarity index 82% rename from tbxi/prclc.py rename to tbxi/parcels_build.py index 6b647ee..472d3b8 100644 --- a/tbxi/prclc.py +++ b/tbxi/parcels_build.py @@ -10,6 +10,9 @@ try: except ImportError: from .slow_lzss import compress +from . import dispatcher + + class CodeLine(dict): def __getattr__(self, attrname): return self[attrname] @@ -56,42 +59,10 @@ def getbool(from_str): class PdslParseError(Exception): pass -def load_and_cache_path(from_path): - # No compression, easy - if not from_path.lower().endswith('.lzss'): - with open(from_path, 'rb') as f: - return f.read() - - # Compression, first try to read cached file - try: - f = open(from_path, 'rb') - except FileNotFoundError: - pass - else: - try: - orig_t = path.getmtime(from_path[:-5]) - except FileNotFoundError: - orig_t = None - - if orig_t is None or orig_t < path.getmtime(from_path): - data = f.read() - f.close() - return data - - # Compression, no valid cached file available - with open(from_path[:-5], 'rb') as f: - data = compress(f.read()) - - with open(from_path, 'wb') as f: - f.write(data) - - return data - -def compile(src): - parent = path.dirname(path.abspath(src)) +def build(src): node_list = [] - with open(src) as f: + with open(path.join(src, 'Parcelfile')) as f: try: for line_num, line in enumerate(f, start=1): level = get_indent_level(line) @@ -117,10 +88,14 @@ def compile(src): if not path.isabs(new.src): # look rel to Parcelfile new.src = path.join(path.dirname(src), new.src) - if new.src.lower().endswith('.lzss'): + a, b = path.splitext(new.src) + if b.lower() == '.lzss': + new.src = a new.compress = 'lzss' - new.data = load_and_cache_path(new.src) + new.data = dispatcher.build(new.src) + if new.compress == 'lzss': + new.data = compress(new.data) node_list[-1].children.append(new) diff --git a/tbxi/prcldump.py b/tbxi/parcels_dump.py similarity index 90% rename from tbxi/prcldump.py rename to tbxi/parcels_dump.py index 9067185..a07edd2 100644 --- a/tbxi/prcldump.py +++ b/tbxi/parcels_dump.py @@ -4,7 +4,9 @@ from os import path from shlex import quote import struct -from .lzss import decompress +from . import dispatcher + +from .slow_lzss import decompress from .lowlevel import PrclNodeStruct, PrclChildStruct def walk_tree(binary): @@ -13,9 +15,6 @@ def walk_tree(binary): e.g. [(prclnodetuple, [prclchildtuple, ...]), ...] """ - if not binary.startswith(b'prcl'): - raise ValueError('binary does not start with magic number') - prclnode = None parents = [] @@ -41,7 +40,7 @@ def suggest_names_to_dump(parent, child, code_name): # We yield heaps of suggested filenames, and the shortest non-empty unique one gets chosen if parent.ostype == child.ostype == 'rom ': - yield 'ROM' + yield 'MacROM' return if 'AAPL,MacOS,PowerPC' in child.name and code_name == 'PowerMgrPlugin': @@ -112,9 +111,14 @@ def settle_name_votes(vote_dict): return decision -def dump(binary, dest, dest_dir): - if path.isdir(dest) or dest.endswith(os.sep): - dest = path.join(dest, 'Parcelfile') +def is_parcels(binary): + return binary.startswith(b'prcl') + + +def dump(binary, dest_dir): + if not binary.startswith(b'prcl'): raise dispatcher.WrongFormat + + os.makedirs(dest_dir, exist_ok=True) basic_structure = walk_tree(binary) @@ -145,6 +149,8 @@ def dump(binary, dest, dest_dir): for prclchild in children: if prclchild.ostype in ('cstr', 'csta'): continue votes = suggest_names_to_dump(prclnode, prclchild, code_name) + if unpacked_dict[unique_binary_tpl(prclchild)].startswith(b'Joy!'): + votes = [v + '.pef' for v in votes] name_vote_dict[unique_binary_tpl(prclchild)].extend(votes) # Decide on filenames @@ -152,11 +158,14 @@ def dump(binary, dest, dest_dir): # Dump blobs to disk for tpl, filename in decision.items(): - with open(path.join(dest_dir, filename), 'wb') as f: - f.write(unpacked_dict[tpl]) + keep_this = True + + data = unpacked_dict[tpl] + dispatcher.dump(data, path.join(dest_dir, filename)) + # Get printing!!! - with open(dest, 'w') as f: + with open(path.join(dest_dir, 'Parcelfile'), 'w') as f: for prclnode, children in basic_structure: line = quote(prclnode.ostype) line += ' flags=0x%05x' % prclnode.flags @@ -173,7 +182,7 @@ def dump(binary, dest, dest_dir): if prclchild.ostype not in ('cstr', 'csta'): filename = decision[unique_binary_tpl(prclchild)] if prclchild.compress == 'lzss': filename += '.lzss' - line += ' src=%s' % quote(path.relpath(path.join(dest_dir, filename), path.dirname(dest))) + line += ' src=%s' % filename if binary_counts[unique_binary_tpl(prclchild)] > 1: line += ' deduplicate=1' diff --git a/tbxi/powerpc_dump.py b/tbxi/powerpc_dump.py new file mode 100644 index 0000000..7451716 --- /dev/null +++ b/tbxi/powerpc_dump.py @@ -0,0 +1,337 @@ +from .lowlevel import SuperMarioHeader, ConfigInfo + +import struct + +import os +from os import path + +from . import dispatcher + + +PAD = b'kc' * 100 + +CONFIGINFO_TEMPLATE = """ +ROMImageBaseOffset= # [28] Offset of Base of total ROM image +ROMImageSize= # [2C] Number of bytes in ROM image +ROMImageVersion= # [30] ROM Version number for entire ROM + + # ROM component Info (offsets are from base of ConfigInfo page) +Mac68KROMOffset= # [34] Offset of base of Macintosh 68K ROM +Mac68KROMSize= # [38] Number of bytes in Macintosh 68K ROM + +ExceptionTableOffset= # [3C] Offset of base of PowerPC Exception Table Code +ExceptionTableSize= # [40] Number of bytes in PowerPC Exception Table Code + +HWInitCodeOffset= # Offset of base of Hardware Init Code +HWInitCodeSize= # [48] Number of bytes in Hardware Init Code + +KernelCodeOffset= # [4C] Offset of base of NanoKernel Code +KernelCodeSize= # [50] Number of bytes in NanoKernel Code + +EmulatorCodeOffset= # [54] Offset of base of Emulator Code +EmulatorCodeSize= # [58] Number of bytes in Emulator Code + +OpcodeTableOffset= # [5C] Offset of base of Opcode Table +OpcodeTableSize= # [60] Number of bytes in Opcode Table + + # Offsets within the Emulator Data Page. +BootstrapVersion= # [64] Bootstrap loader version info +############## VALUES SPECIFIC TO THIS 68K EMULATOR VERSION ############## +BootVersionOffset= # [74] offset within EmulatorData of BootstrapVersion +ECBOffset= # [78] offset within EmulatorData of ECB +IplValueOffset= # [7C] offset within EmulatorData of IplValue + + # Offsets within the Emulator Code. +EmulatorEntryOffset= # [80] offset within Emulator Code of entry point +KernelTrapTableOffset= # [84] offset within Emulator Code of KernelTrapTable + + # Interrupt Passing Masks. +TestIntMaskInit= # [88] initial value for test interrupt mask +ClearIntMaskInit= # [8C] initial value for clear interrupt mask +PostIntMaskInit= # [90] initial value for post interrupt mask +##################### END OF EMULATOR-SPECIFIC VALUES #################### +LA_InterruptCtl= # [94] logical address of Interrupt Control I/O page +InterruptHandlerKind= # [98] kind of handler to use + +LA_InfoRecord= # [9C] logical address of InfoRecord page +LA_KernelData= # [A0] logical address of KernelData page +LA_EmulatorData= # [A4] logical address of EmulatorData page +LA_DispatchTable= # [A8] logical address of Dispatch Table +LA_EmulatorCode= # [AC] logical address of Emulator Code + + # Address Space Mapping. +PageAttributeInit= # [B4] default WIMG/PP settings for PTE creation + + # Only needed for Smurf +SharedMemoryAddr= # [35C] physical address of Mac/Smurf shared message mem + +PA_RelocatedLowMemInit= # [360] physical address of RelocatedLowMem + +OpenFWBundleOffset= # [364] Offset of base of OpenFirmware PEF Bundle +OpenFWBundleSize= # [368] Number of bytes in OpenFirmware PEF Bundle + +LA_OpenFirmware= # [36C] logical address of Open Firmware +PA_OpenFirmware= # [370] physical address of Open Firmware +LA_HardwarePriv= # [374] logical address of HardwarePriv callback +""".strip() + + +def extract_and_zero(binary, start, stop): + ret = binary[start:stop] + binary[start:stop] = bytes(stop - start) + return ret + + +def find_configinfo(binary): + # Find a ConfigIngo struct by checking every possible + # place for a valid checksum. Ugly but quick. + + byte_lanes = [sum(binary[i::8]) for i in range(8)] + + for i in range(0, len(binary), 0x100): + zeroed_byte_lanes = list(byte_lanes) + for j in range(i, i+40): + zeroed_byte_lanes[j % 8] -= binary[j] + + sum32 = [lane % (1<<32) for lane in zeroed_byte_lanes] + + sum64 = sum(lane << (k * 8) for (k, lane) in enumerate(reversed(zeroed_byte_lanes))) + sum64 %= 1 << 64 + + allsums = b''.join(x.to_bytes(4, byteorder='big') for x in sum32) + allsums += sum64.to_bytes(8, byteorder='big') + + if binary[i:i+len(allsums)] == allsums: + break + else: + return + + # Which structs share the BootstrapVersion signature? + for j in range(0, len(binary), 0x100): + if binary[i+0x64:i+0x74] == binary[j+0x64:j+0x74]: + yield j + + +def dump_configinfo(binary, offset, push_line): + s = ConfigInfo.unpack_from(binary, offset) + + # First section (no [header]): + # Raw key=value lines not resembling the struct in PCCInfoRecordsPriv.h + for line in CONFIGINFO_TEMPLATE.split('\n'): + if '=' in line: + key, _, remainder = line.partition('=') + raw_value = getattr(s, key) + if key == 'InterruptHandlerKind': + value = '0x%02X' % raw_value + elif key == 'BootstrapVersion': + value = repr(raw_value)[1:] + elif key.endswith('Offset') and key.startswith(('Mac68KROM', 'ExceptionTable', 'HWInitCode', 'KernelCode', 'EmulatorCode', 'OpcodeTable', 'OpenFWBundle')): + if getattr(s, key.replace('Offset', 'Size')) == 0: + value = '0x00000000' + else: + value = 'BASE0x%+X' % (raw_value - s.ROMImageBaseOffset) + else: + value = '0x%08X' % raw_value + + value = value.replace('0x-', '-0x').replace('0x+', '+0x') + + nuline = key + '=' + value + while remainder.startswith(' ') and len(nuline) + len(remainder) > len(line): + remainder = remainder[1:] + nuline += remainder + line = nuline + + push_line(line) + + push_line('') + + # Now dump the more structured parts of the ConfigInfo + mapnames = ['sup', 'usr', 'cpu', 'ovl'] + + segmaps = [[],[],[],[]] + for i, blob in enumerate((s.SegMap32SupInit, s.SegMap32UsrInit, s.SegMap32CPUInit, s.SegMap32OvlInit)): + for j in range(0, len(blob), 8): + tpl = struct.unpack_from('>LL', blob, j) + segmaps[i].append(tpl) + + def print_seg_ptrs_for_offset(segmap_offset): + for header, list16 in zip(mapnames, segmaps): + for seg_i, (seg_offset, seg_reg) in enumerate(list16): + if seg_offset == segmap_offset: + push_line('segment_ptr_here=0x%X map=%s segment_register=0x%08X' % (seg_i, header, seg_reg)) + + batmaps = [[],[],[],[]] + for i, blob in enumerate((s.BatMap32SupInit, s.BatMap32UsrInit, s.BatMap32CPUInit, s.BatMap32OvlInit)): + for j in reversed(range(0, 32, 4)): + batmaps[i].append(((blob >> j) & 0xF) * 8) + + last_used_batmap = max(y for x in batmaps for y in x) + + def print_bat_ptrs_for_offset(batmap_offset): + for header, list8 in zip(mapnames, batmaps): + for bat_offset, bat_name in zip(list8, ['ibat0', 'ibat1', 'ibat2', 'ibat3', 'dbat0', 'dbat1', 'dbat2', 'dbat3']): + if bat_offset == batmap_offset: + push_line('bat_ptr_here=%s map=%s' % (bat_name, header)) + + lowmem = [] + lmoffset = s.MacLowMemInitOffset + while any(binary[offset+lmoffset:][:4]): + key, val = struct.unpack_from('>LL', binary, offset+lmoffset) + lowmem.append((key, val)) + lmoffset += 8 + + push_line('[LowMemory]') + for key, val in lowmem: + push_line('address=0x%08X value=0x%08X' % (key, val)) + push_line('') + + push_line('[PageMappingInfo]') + if s.PageMapInitSize or any(s.SegMap32SupInit + s.SegMap32UsrInit + s.SegMap32CPUInit + s.SegMap32OvlInit): + push_line('# Constants: PMDT_InvalidAddress = 0xA00, PMDT_Available = 0xA01') + + pagemapinit = binary[offset:][s.PageMapInitOffset:][:s.PageMapInitSize] + + for i in range(0, len(pagemapinit), 8): + print_seg_ptrs_for_offset(i) + + pgidx, pgcnt, word2 = struct.unpack_from('>HHL', pagemapinit, i) + attr = word2 & 0xFFF + + if attr == 0xA00: + attr_s = 'PMDT_InvalidAddress' + elif attr == 0xA01: + attr_s = 'PMDT_Available' + else: + attr_s = '0x%03X' % attr + + paddr = word2 >> 12 + if 'Rel' in attr_s: + paddr_s = 'BASE+0x%05X' % ((paddr + offset) & 0xFFFFF) + else: + paddr_s = '0x%05X' % paddr + + if i == s.PageMapIRPOffset: push_line('special_pmdt=irp') + if i == s.PageMapKDPOffset: push_line('special_pmdt=kdp') + if i == s.PageMapEDPOffset: push_line('special_pmdt=edp') + + push_line('pmdt_page_offset=0x%04X pages_minus_1=0x%04X phys_page=%s attr=%s' % (pgidx, pgcnt, paddr_s, attr_s)) + + push_line('') + + push_line('[BatMappingInfo]') + if any(s.BATRangeInit) or s.BatMap32SupInit or s.BatMap32UsrInit or s.BatMap32CPUInit or s.BatMap32OvlInit: + for i in range(0, len(s.BATRangeInit), 8): + if i > last_used_batmap * 8: break + + print_bat_ptrs_for_offset(i) + + u, l = struct.unpack_from('>LL', s.BATRangeInit, i) + + is_relative = l & 0x200 + if is_relative: + l = (offset + l) & 0xFFFFFFFF - is_relative + + bepi = u >> 17 + bl = (u >> 2) & 0x7FF + vs = (u >> 1) & 1 + vp = u & 1 + + brpn = l >> 17 + wimg = [(l > 6) & 1, (l > 5) & 1, (l > 4) & 1, (l > 3) & 1] + pp = [(l > 1) & 1, l & 1] + + bl_s = '0b' + bin(bl)[2:].zfill(11) + + if is_relative: + brpn_s = 'BASE+0x%06X' % (brpn << 17) + else: + brpn_s = '0x%08X' % (brpn << 17) + + push_line('bepi=0x%08X bl=%s vs=%s vp=%d brpn=%s wimg=0b%d%d%d%d pp=0b%d%d' % (bepi << 17, bl_s, vs, vp, brpn_s, *wimg, *pp)) + + push_line('') + + + +def is_powerpc(binary): + return (len(binary) == 0x400000) and (PAD in binary[:0x300000]) + + +def get_nk_version(nk): + if nk.startswith(b'\x48\x00\x00\x0C'): + # v2 NK has structured header + return 'v%02X.%02X' % (nk[4], nk[5]) + + for i in range(0, len(nk) - 8, 4): + if nk[i:i+2] == b'\x39\x80': # li r12, ??? + if nk[i+4:i+8] == b'\xB1\x81\x0F\xE4': # sth r12, 0xFE4(r1) + return 'v%02X.%02X' % (nk[i+2], nk[i+3]) # return the ??? + + +def extract_plausible_thing(binary, start): + stop = binary.find(bytes(1024), start) # check, because kernel is often absent or wrong size + if stop > start: + while stop % 4 != 0: stop += 1 + return extract_and_zero(binary, start, stop) + + +def dump(binary, dest_dir): + if not is_powerpc(binary): raise dispatcher.WrongFormat + + os.makedirs(dest_dir, exist_ok=True) + + cioffsets = list(find_configinfo(binary)) + + # We will zero out parts as we go along extracting them + binary = bytearray(binary) + + for i, cioffset in enumerate(cioffsets, 1): + filename = 'Configfile' + if len(cioffsets) > 1: filename += '-' + str(i) + + with open(path.join(dest_dir, filename), 'w') as f: + push_line = lambda x: print(x, file=f) + dump_configinfo(binary, cioffset, push_line) + + best_cioffset = cioffsets[0] + best_ci = ConfigInfo.unpack_from(binary, best_cioffset) + + for cioffset in cioffsets: + extract_and_zero(binary, cioffset, cioffset + 0x1000) + + supermario = extract_and_zero(binary, + best_cioffset + best_ci.Mac68KROMOffset, + best_cioffset + best_ci.Mac68KROMOffset + best_ci.Mac68KROMSize) + dispatcher.dump(supermario, path.join(dest_dir, 'Mac68KROM')) + + xtbl = extract_and_zero(binary, + best_cioffset + best_ci.ExceptionTableOffset, + best_cioffset + best_ci.ExceptionTableOffset + best_ci.ExceptionTableSize) + if any(xtbl): + # xtbl_len = len(xtbl) + # while not any(xtbl[xtbl_len-4:xtbl_len]): xtbl_len -= 4 + # xtbl = xtbl[xtbl_len:] + with open(path.join(dest_dir, 'ExceptionTable'), 'wb') as f: + f.write(xtbl) + + nk = extract_plausible_thing(binary, min(0x310000, best_cioffset + best_ci.KernelCodeOffset)) + if nk: + name = 'NanoKernel' + vers = get_nk_version(nk) + if vers: name += '-' + vers + + with open(path.join(dest_dir, name), 'wb') as f: + f.write(nk) + + hwinit = extract_plausible_thing(binary, best_cioffset + best_ci.HWInitCodeOffset) + if hwinit: + with open(path.join(dest_dir, 'HWInit'), 'wb') as f: + f.write(hwinit) + + openfw = extract_plausible_thing(binary, best_cioffset + best_ci.OpenFWBundleOffset) + if openfw: + with open(path.join(dest_dir, 'OpenFW'), 'wb') as f: + f.write(openfw) + + with open(path.join(dest_dir, 'EverythingElse'), 'wb') as f: + f.write(binary) diff --git a/tbxi/supermario_dump.py b/tbxi/supermario_dump.py new file mode 100644 index 0000000..3012f86 --- /dev/null +++ b/tbxi/supermario_dump.py @@ -0,0 +1,161 @@ +from os import path +import os +import shlex + +from .lowlevel import SuperMarioHeader, ResHeader, ResEntry, FakeMMHeader, COMBO_FIELDS + +from . import dispatcher + + +PAD = b'kc' * 100 + +HEADER_COMMENT = """ +# Automated dump of Macintosh ROM resources + +# The (optional) combo mask switches a resource based on the DefaultRSRCs +# field of the box's ProductInfo structure. (The low-memory variable at +# 0xDD8 points to ProductInfo, and the DefaultRSRCs byte is at offset +# 0x16.) The combo field is usually used for the Standard Apple Numeric +# Environment (SANE) PACKs 4 and 5. + +# Summary of known combos: +# 0b01111000 AllCombos (DEFAULT) Universal resource +# 0b01000000 AppleTalk1 Appletalk 1.0 +# 0b00100000 AppleTalk2 Appletalk 2.0 +# 0b00110000 AppleTalk2_NetBoot_FPU Has FPU and remote booting +# 0b00001000 AppleTalk2_NetBoot_NoFPU Has remote booting, no FPU +# 0b00010000 NetBoot Has remote booting + +""".strip() + +def clean_maincode(binary): + binary = bytearray(binary) + + header = SuperMarioHeader.unpack_from(binary) + modified_header = header._asdict() + + for k in list(modified_header): + if k.startswith('CheckSum'): + modified_header[k] = 0 + modified_header['RomRsrc'] = 0 + modified_header['RomSize'] = 1 + + SuperMarioHeader.pack_into(binary, 0, **modified_header) + + return bytes(binary) + + +def is_supermario(binary): + return (len(binary) in (0x200000, 0x300000)) and (PAD in binary) + + +def extract_decldata(binary): + return binary[binary.rfind(PAD) + len(PAD):] + + +def extract_resource_offsets(binary): + # chase the linked list around + offsets = [] + + reshead = SuperMarioHeader.unpack_from(binary).RomRsrc + link = ResHeader.unpack_from(binary, reshead).offsetToFirst + while link: + offsets.append(link) + link = ResEntry.unpack_from(binary, link).offsetToNext + + offsets.reverse() + return offsets + + +def sanitize_macroman(binary): + string = binary.decode('mac_roman') + string = ''.join(c if c.isalpha() or c.isdigit() else '_' for c in string) + return string + + +def express_macroman(binary): + return repr(binary)[1:] + accum = '' + for b in binary: + if b == ord(' '): + accum += '\\ ' + elif b < 128 and chr(b).isprintable() and not chr(b).isspace(): + accum += chr(b) + else: + accum += '\\x%02X' % b + return accum + + +def quodec(binary): + return shlex.quote(binary.decode('mac_roman')) + + +def ljustspc(s, n): + return (s + ' ').ljust(n) + + +def dump(binary, dest_dir): + if not is_supermario(binary): raise dispatcher.WrongFormat + + os.makedirs(dest_dir, exist_ok=True) + + header = SuperMarioHeader.unpack_from(binary) + + main_code = clean_maincode(binary[:header.RomRsrc]) + with open(path.join(dest_dir, 'MainCode'), 'wb') as f: + f.write(main_code) + + decldata = extract_decldata(binary) + with open(path.join(dest_dir, 'DeclData'), 'wb') as f: + f.write(decldata) + + # now for the tricky bit: resources :( + f = open(path.join(dest_dir, 'Rsrcfile'), 'w') + print(HEADER_COMMENT + '\n', file=f) + + unavail_filenames = set(['', '.pef']) + + for i, offset in enumerate(extract_resource_offsets(binary)): + rsrc_dir = path.join(dest_dir, 'Rsrc') + os.makedirs(rsrc_dir, exist_ok=True) + + entry = ResEntry.unpack_from(binary, offset) + mmhead = FakeMMHeader.unpack_from(binary, entry.offsetToData - FakeMMHeader.size) + + # assert entry. + assert mmhead.MagicKurt == b'Kurt' + assert mmhead.MagicC0A00000 == 0xC0A00000 + + data = binary[entry.offsetToData:][:mmhead.dataSizePlus12 - 12] + report_combo_field = COMBO_FIELDS.get(entry.combo, '0b' + bin(entry.combo >> 56)[2:].zfill(8)) + + # create a friendly ascii filename for the resource + filename = '%s_%d' % (sanitize_macroman(entry.rsrcType), entry.rsrcID) + if len(entry.rsrcName) > 0 and entry.rsrcName != b'Main': # uninformative artifact of rom build + filename += '_' + sanitize_macroman(entry.rsrcName) + if report_combo_field != 'AllCombos': + filename += '_' + report_combo_field.replace('AppleTalk', 'AT') + filename = filename.strip('_') + while '__' in filename: filename = filename.replace('__', '_') + if data.startswith(b'Joy!peff'): filename += '.pef' + while filename in unavail_filenames: filename = '_' + filename + + unavail_filenames.add(filename) + + with open(path.join(rsrc_dir, filename), 'wb') as f2: + f2.write(data) + + filename = path.join('Rsrc', filename) + + # Now, just need to dream up a data format + report = '' + report = ljustspc(report + 'type=' + quodec(entry.rsrcType), 12) + report = ljustspc(report + 'id=' + str(entry.rsrcID), 24) + report = ljustspc(report + 'name=' + quodec(entry.rsrcName), 48) + report = ljustspc(report + 'src=' + shlex.quote(filename), 84) + if report_combo_field != 'AllCombos': + report = ljustspc(report + 'combo=' + report_combo_field, 0) + report = report.rstrip() + + print(report, file=f) + diff --git a/tests/test_lzss.py b/tests/test_lzss.py new file mode 100644 index 0000000..03b2c5e --- /dev/null +++ b/tests/test_lzss.py @@ -0,0 +1,13 @@ +from tbxi.slow_lzss import decompress +from tbxi.fast_lzss import compress +import random + +def test_random(): + the_len = 0 + while the_len < 4 * 1024 * 1024: + the_len <<= 1 + the_len |= random.choice((1, 0)) + + tryout = bytes(random.choice(range(256)) for x in range(the_len)) + + assert decompress(compress(tryout)) == tryout