Support fixed-offset resources

Most TNT-era ROMs use this feature of RomLayout. Heuristically detect,
report and rebuild these ROMs.

tbxi/supermario_build.py

@@ -37,7 +37,7 @@ def build(src):
             if sep == '=':
                 if a == 'type':
                     b = b.encode('mac_roman')
-                elif a in ('id', 'rom_size'):
+                elif a in ('id', 'rom_size', 'offset'):
                     b = ast.literal_eval(b)
 
                 thisdict[a] = b
@@ -48,39 +48,70 @@ def build(src):
             rsrc_list.append(thisdict)
 
     rom = bytearray(b'kc' * (rom_size // 2))
+    free_map = bytearray(b'X' * (rom_size // ALIGN))
+
+    def rom_insert(offset, binary, letter=' '):
+        letter = letter.encode('ascii')
+
+        if offset + len(binary) > len(rom):
+            raise IndexError('ROM too small to insert %r at %s' % (letter, hex(offset)))
+        rom[offset:offset+len(binary)] = binary
+
+        # Populate the informational array of letters
+        # Only capitals can be overwritten
+        start = offset // ALIGN
+        stop = (offset + len(binary) - 1) // ALIGN + 1
+
+        for i in range(start, stop):
+            if free_map[i] != 20:
+                if free_map[i:i+1].upper() != free_map[i:i+1]:
+                    raise ValueError('Tried to insert %r over %r at %s' % (letter, free_map[i], hex(offset)))
+
+            free_map[i:i+1] = letter
+
+    def find_free(length):
+        length = (length + ALIGN - 1) // ALIGN
+        return free_map.index(b'X' * length) * ALIGN
 
     maincode = dispatcher.build(path.join(src, 'MainCode'))
-    rom[:len(maincode)] = maincode
+    rom_insert(0, maincode, 'm')
+
+    head_ptr = find_free(16)
+    rom_insert(head_ptr, b'fake header', 'H')
 
     try:
         decldata = dispatcher.build(path.join(src, 'DeclData'))
     except FileNotFoundError:
         pass
     else:
-        if decldata: rom[-len(decldata):] = decldata
+        rom_insert(len(rom) - len(decldata), decldata, 'd')
 
     # now blat in the resources
-    free = len(maincode) + 16
+    ent_ptr = 0
-    prev_ent_ptr = 0
     bogus_off = 0x5C
 
     for r in rsrc_list:
         data = dispatcher.build(path.join(src, r['src']))
 
-        data_ptr = lowlevel.pad(free + 16, ALIGN)
+        # First place the data, including the fake MemMgr header
-        mm_ptr = data_ptr - 12
+        if 'offset' in r:
-        ent_ptr = lowlevel.pad(data_ptr + len(data), ALIGN)
+            ofs = r['offset']
+        else:
+            ofs = find_free(16 + len(data))
 
+        mm_ptr = ofs + 4
+        data_ptr = ofs + 16
+
+        # And insert that
         mm = lowlevel.FakeMMHeader.pack(
             MagicKurt=b'Kurt',
             MagicC0A00000=0xC0A00000,
             dataSizePlus12=len(data) + 12,
             bogusOff=bogus_off,
         )
-        rom[mm_ptr-4:mm_ptr-4+len(mm)] = mm
+        rom_insert(mm_ptr - 4, mm + data, 'r')
-
-        rom[data_ptr:data_ptr+len(data)] = data
 
+        # Calculate the independently-placed entry struct
         combo = r.get('combo', 'AllCombos')
         combo = REV_COMBO_FIELDS.get(combo, None)
         if combo is None:
@@ -88,7 +119,7 @@ def build(src):
 
         ent = lowlevel.ResEntry.pack(
             combo=combo,
-            offsetToNext=prev_ent_ptr,
+            offsetToNext=ent_ptr, # this is the previous one
             offsetToData=data_ptr,
             rsrcType=r['type'],
             rsrcID=r['id'],
@@ -98,24 +129,24 @@ def build(src):
         # snip off pascal string padding
         ent = ent[:0x18 + ent[0x17]]
 
-        rom[ent_ptr:ent_ptr+len(ent)] = ent
+        # Place the entry struct
+        ent_ptr = find_free(len(ent))
+        rom_insert(ent_ptr, ent, 'e')
 
-        free = ent_ptr + len(ent)
-        prev_ent_ptr = ent_ptr
         bogus_off += 8
 
     head = lowlevel.ResHeader.pack(
-        offsetToFirst=prev_ent_ptr,
+        offsetToFirst=ent_ptr,
         maxValidIndex=4,
         comboFieldSize=8,
         comboVersion=1,
         headerSize=12,
     )
-    rom[len(maincode):len(maincode)+len(head)] = head
+    rom_insert(head_ptr, head, 'h')
 
     # now set the size
     fields = lowlevel.SuperMarioHeader.unpack_from(rom)._asdict()
-    fields['RomRsrc'] = len(maincode)
+    fields['RomRsrc'] = head_ptr
     fields['RomSize'] = len(rom)
     lowlevel.SuperMarioHeader.pack_into(rom, 0, **fields)
 

tbxi/supermario_dump.py

@@ -53,6 +53,7 @@ def extract_decldata(binary):
     return binary[binary.rfind(PAD) + len(PAD):]
 
 
+# Get a list of (entry_offset, data_offset, data_len)
 def extract_resource_offsets(binary):
     # chase the linked list around
     offsets = []
@@ -60,7 +61,9 @@ def extract_resource_offsets(binary):
     reshead = SuperMarioHeader.unpack_from(binary).RomRsrc
     link = ResHeader.unpack_from(binary, reshead).offsetToFirst
     while link:
-        offsets.append(link)
+        data = ResEntry.unpack_from(binary, link).offsetToData
+        datasize = FakeMMHeader.unpack_from(binary, data - 16).dataSizePlus12 - 12
+        offsets.append((link, data, datasize))
         link = ResEntry.unpack_from(binary, link).offsetToNext
 
     offsets.reverse()
@@ -116,18 +119,31 @@ def dump(binary, dest_dir):
         unavail_filenames = set(['', '.pef', '.pict'])
         types_where_Main_should_be_in_filename = set()
 
-        for i, offset in enumerate(extract_resource_offsets(binary)):
+        known_forced_offsets = []
+
+        for i, (hoffset, doffset, dlen) in enumerate(extract_resource_offsets(binary)):
             rsrc_dir = path.join(dest_dir, 'Rsrc')
             os.makedirs(rsrc_dir, exist_ok=True)
 
-            entry = ResEntry.unpack_from(binary, offset)
+            data = binary[doffset:doffset+dlen]
-            mmhead = FakeMMHeader.unpack_from(binary, entry.offsetToData - FakeMMHeader.size)
+            entry = ResEntry.unpack_from(binary, hoffset)
 
-            # assert entry.
+            offset_forced = False
-            assert mmhead.MagicKurt == b'Kurt'
+            if hoffset < doffset: # Either offset was forced, or previous forced offset caused fit problems
-            assert mmhead.MagicC0A00000 == 0xC0A00000
+                # Tricky guessing: check whether the data would have fit where the entry struct went
+                for known in known_forced_offsets:
+                    if hoffset <= known < hoffset + 16 + dlen:
+                        break
+                else:
+                    offset_forced = True
+
+            if offset_forced:
+                known_forced_offsets.append(doffset - 16)
+
+            # mmhead = FakeMMHeader.unpack_from(binary, doffset - 16)
+            # assert mmhead.MagicKurt == b'Kurt'
+            # assert mmhead.MagicC0A00000 == 0xC0A00000
             
-            data = binary[entry.offsetToData:][:mmhead.dataSizePlus12 - 12]
             report_combo_field = COMBO_FIELDS.get(entry.combo, '0b' + bin(entry.combo >> 56)[2:].zfill(8))
 
             if entry.rsrcName == b'%A5Init':
@@ -160,6 +176,8 @@ def dump(binary, dest_dir):
             report = ljustspc(report + 'src=' + shlex.quote(filename), 84)
             if report_combo_field != 'AllCombos':
                 report = ljustspc(report + 'combo=' + report_combo_field, 0)
+            if offset_forced:
+                report = ljustspc(report + 'offset=0x%X' % (doffset - 16), 0)
             report = report.rstrip()
 
             print(report, file=f)