build improvements (no binary changes)

Makefile

@@ -176,10 +176,10 @@ index: preconditions md asmfx asmprelaunch asmdemo compress extract
 # in the form of OKVS data structures, plus game counts in the form of source files
 #
 	[ -f build/index ] || $(PARALLEL) ::: \
-	    '(grep "^00" < build/GAMES.CONF | bin/buildsearch.sh src/index/count00.a build/HGR.TITLES.LOG /dev/null > build/SEARCH00.IDX)' \
-	    '(grep "^0" < build/GAMES.CONF | bin/buildsearch.sh src/index/count01.a build/HGR.TITLES.LOG build/DHGR.TITLES.LOG > build/SEARCH01.IDX)' \
-	    '(grep "^.0" < build/GAMES.CONF | bin/buildsearch.sh src/index/count10.a build/HGR.TITLES.LOG /dev/null > build/SEARCH10.IDX)' \
-	    '(bin/buildsearch.sh src/index/count11.a build/HGR.TITLES.LOG build/DHGR.TITLES.LOG < build/GAMES.CONF > build/SEARCH11.IDX)'
+	    '(grep "^00" < build/GAMES.CONF | bin/buildsearch.py src/index/count00.a build/HGR.TITLES.LOG "" > build/SEARCH00.IDX)' \
+	    '(grep "^0" < build/GAMES.CONF | bin/buildsearch.py src/index/count01.a build/HGR.TITLES.LOG build/DHGR.TITLES.LOG > build/SEARCH01.IDX)' \
+	    '(grep "^.0" < build/GAMES.CONF | bin/buildsearch.py src/index/count10.a build/HGR.TITLES.LOG "" > build/SEARCH10.IDX)' \
+	    '(bin/buildsearch.py src/index/count11.a build/HGR.TITLES.LOG build/DHGR.TITLES.LOG < build/GAMES.CONF > build/SEARCH11.IDX)'
 #
 # add IDX files to the combined index file and generate
 # the index records that callers use to reference them

bin/buildsearch.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+# parameters
+# stdin - input containing list of game metadata, filename, display name (e.g. GAMES.CONF or some subset of it)
+# stdout - binary OKVS data structure
+# 1 - output filename for game count in assembler code format
+# 2 - input filename of HGR titles, offsets, and sizes
+# 3 - input filename of DHGR titles, offsets, and sizes
+
+import argparse
+import pprint
+import struct
+import sys
+
+gSearchIndex = 0x6000 # must match gSearchIndex in src/constants.a
+
+# indexes into |flags| as string
+iDHGRTitle = 2
+iCheatCategory = 3
+iSingleLoad = 4
+
+# maps of flag raw string value -> value in final flags byte
+kHasDHGRTitle =  {'0': 0, '1': 128}
+kSingleLoad =    {'0': 0, '1': 64}
+
+def parse_log_file(filename):
+    rv = {}
+    if filename:
+        with open(filename, 'r') as f:
+            lines = [x.strip().split(',') for x in f.readlines()]
+        for title, offset, size in lines:
+            rv[title] = (int(offset), int(size))
+    return rv
+
+def build(records, args):
+    # records is [(flags, key, value), (flags, key, value) ...]
+    hgr_cache = parse_log_file(args.input_hgr_log_file)
+    dhgr_cache = parse_log_file(args.input_dhgr_log_file)
+    cache_ptr = {'0': hgr_cache, '1': dhgr_cache}
+    record_count = len(records)
+
+    # generate source file with game count
+    with open(args.output_game_count_file, 'w') as file_handle:
+        file_handle.write(f""";
+; Game count
+;
+; This file is automatically generated
+;
+!word {record_count:>8}
+""")
+
+    # yield OKVS record count (2 bytes, unsigned int, little-endian)
+    yield struct.pack('<H', record_count)
+
+    # yield OKVS lookup table address (2 bytes, unsigned int, little-endian)
+    # lookup table is stored after all record data, so first calculate total record size
+    # record_count * (length-prefixed key + length-prefixed value + 8 other bytes)
+    # then lookup table address is that + gSearchIndex + 4 bytes for the OKVS header
+    total_record_size = len("".join([x for xs in records for x in xs[1:]])) + 10*record_count
+    yield struct.pack('<H', total_record_size + gSearchIndex + 4)
+
+    rec_key_address = gSearchIndex + 5
+    key_addresses = []
+    for flags, key, value in records:
+        key_addresses.append(rec_key_address)
+        rec_length = len(key) + len(value) + 10
+        rec_key_address += rec_length
+
+        # yield record length (1 byte)
+        yield struct.pack('B', rec_length)
+
+        # yield key (Pascal-style string)
+        yield struct.pack(f'{len(key)+1}p', key.encode('ascii'))
+
+        # yield value (Pascal-style string)
+        yield struct.pack(f'{len(value)+1}p', value.encode('ascii'))
+
+        yield struct.pack('B', 1)
+
+        # yield flags
+        has_dhgr_title = dhgr_cache and flags[iDHGRTitle] or '0'
+        yield struct.pack('B', kHasDHGRTitle[has_dhgr_title] + \
+                          kSingleLoad[flags[iSingleLoad]] + \
+                          int(flags[iCheatCategory]))
+
+        rec_offset, rec_size = cache_ptr[has_dhgr_title][key]
+
+        # yield record offset (3 bytes, big-endian, unsigned long)
+        yield struct.pack('>L', rec_offset)[1:]
+
+        # yield record size (2 bytes, little-endian, unsigned short)
+        yield struct.pack('<H', rec_size)
+
+    # yield lookup table
+    yield struct.pack(f'<{record_count}H', *key_addresses)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Build indexed OKVS structure for search cache")
+    parser.add_argument("output_game_count_file")
+    parser.add_argument("input_hgr_log_file")
+    parser.add_argument("input_dhgr_log_file")
+    args = parser.parse_args()
+    records = [x.strip() for x in sys.stdin.readlines()]
+    records = [x.replace('=',',').split(',')
+               for x in records
+               if x and x[0] not in ('#', '[')]
+    for b in build(records, args):
+        sys.stdout.buffer.write(b)

bin/buildsearch.sh

@@ -1,77 +0,0 @@
-#!/bin/bash
-
-# parameters
-# stdin - input containing list of game metadata, filename, display name (e.g. GAMES.CONF or some subset of it)
-# stdout - binary OKVS data structure
-# 1 - output filename for game count in assembler code format
-# 2 - input filename of HGR titles, offsets, and sizes
-# 3 - input filename of DHGR titles, offsets, and sizes
-
-# make temp file with just the key/value pairs (strip blank lines, comments, eof marker)
-records=$(mktemp)
-tr -d "\r" | awk '!/^$|^#/' > "$records"
-
-# read logs of offsets & sizes for HGR and DHGR titles
-# that were generated by an earlier script
-hgrlog=$(< "$2")
-dhgrlog=$(< "$3")
-
-# generate source file with game count
-(echo ";"
- echo "; Game count"
- echo ";"
- echo "; This file is automatically generated"
- echo ";"
- echo "!word $(wc -l < "$records")") > "$1"
-
-# make temp assembly source file that represents the binary OKVS data structure
-source=$(mktemp)
-(echo '*=$6000'
- echo "!le16 $(wc -l <"$records")"     # OKVS header
- echo "!word KeyLookup"
- count=0
- while IFS="=" read -r key value; do
-     count=$((count+1))
-     if [ -z "$dhgrlog" ]; then
-         dhgr="0"
-     else
-         dhgr=$(echo "$key" | cut -c3) # 'has DHGR title screen' flag (0 or 1)
-     fi
-     cheat=$(echo "$key" | cut -c4)    # 'cheat category' (0..7)
-     single=$(echo "$key" | cut -c5)   # 'single-load' flag (0 or 1)
-     key=$(echo "$key" | cut -d"," -f2)
-     if [ "$dhgr" -eq "0" ]; then
-         offset=$hgrlog
-         size=$hgrlog
-     else
-         offset=$dhgrlog
-         size=$dhgrlog
-     fi
-     offset=$(echo "$offset" | awk -F, '/^'"$key"',/ { print $2 }')
-     size=$(echo "$size" | awk -F, '/^'"$key"',/ { print $3 }')
-     echo "!byte ${#key}+${#value}+10" # OKVS record length
-     echo "Key${count}"
-     echo "!byte ${#key}"              # OKVS key length
-     echo "!text \"$key\""             # OKVS key (filename)
-     echo "!byte ${#value}"            # OKVS value length
-     echo "!text \"$value\""           # OKVS value (display name)
-     echo "!byte 1"
-     echo "!byte $((dhgr*128))+$((single*64))+$cheat"
-     echo "!be24 $offset"
-     echo "!le16 $size"
- done < "$records"
- echo "KeyLookup"
- for i in $(seq $count); do
-     echo "!word Key$i"
- done
-) > "$source"
-
-# assemble temp source file into binary OKVS data structure, then output that
-out=$(mktemp)
-acme -o "$out" "$source"
-cat "$out"
-
-# clean up
-rm "$out"
-rm "$source"
-rm "$records"