4cade/bin/buildsearch.py

#!/usr/bin/env python3

# parameters
# stdin - input containing list of game metadata, filename, display name (e.g. GAMES.CONF or some subset of it)
# stdout - binary OKVS data structure
# 1 - output filename for game count in assembler code format
# 2 - input filename of HGR titles, offsets, and sizes
# 3 - input filename of DHGR titles, offsets, and sizes

import argparse
import pprint
from struct import pack
import sys

gSearchIndex = 0x6000 # must match gSearchIndex in src/constants.a

# indexes into |flags| as string
iDHGRTitle = 2
iCheatCategory = 3
iSingleLoad = 4

# maps of flag raw string value -> value in final flags byte
kHasDHGRTitle =  {'0': 0, '1': 128}
kSingleLoad =    {'0': 0, '1': 64}

def parse_log_file(filename):
    if not filename:
        return {}
    with open(filename, 'r') as f:
        lines = [x.strip().split(',') for x in f.readlines()]
    lines = [(title, (int(offset), int(size))) for title, offset, size in lines]
    return dict(lines)

def build(records, args):
    # records is [(flags, key, value), (flags, key, value) ...]
    hgr_cache = parse_log_file(args.input_hgr_log_file)
    dhgr_cache = parse_log_file(args.input_dhgr_log_file)
    cache_ptr = {'0': hgr_cache, '1': dhgr_cache}
    record_count = len(records)

    # generate source file with game count
    with open(args.output_game_count_file, 'w') as file_handle:
        file_handle.write(f""";
; Game count
;
; This file is automatically generated
;
!word {record_count:>8}
""")

    # lookup table is stored after all record data, so first calculate total record size
    # record_count * (length-prefixed key + length-prefixed value + 8 other bytes)
    # then lookup table address is that + gSearchIndex + 4 bytes for the OKVS header
    total_record_size = len("".join([x for xs in records for x in xs[1:]])) + 10*record_count

    # yield OKVS header (2 x 2 bytes, unsigned int, little-endian)
    yield pack('<2H', record_count, total_record_size + gSearchIndex + 4)

    rec_key_address = gSearchIndex + 5
    key_addresses = []
    for flags, key, value in records:
        key_addresses.append(rec_key_address)
        rec_length = len(key) + len(value) + 10
        rec_key_address += rec_length

        # yield record length (1 byte)
        yield pack('B', rec_length)

        # yield key (Pascal-style string)
        yield pack(f'{len(key)+1}p', key.encode('ascii'))

        # yield value (Pascal-style string)
        yield pack(f'{len(value)+1}p', value.encode('ascii'))

        yield pack('B', 1)

        # yield flags
        has_dhgr_title = dhgr_cache and flags[iDHGRTitle] or '0'
        yield pack('B', kHasDHGRTitle[has_dhgr_title] + \
                          kSingleLoad[flags[iSingleLoad]] + \
                          int(flags[iCheatCategory]))

        rec_offset, rec_size = cache_ptr[has_dhgr_title][key]

        # yield record offset (3 bytes, big-endian, unsigned long)
        yield pack('>L', rec_offset)[1:]

        # yield record size (2 bytes, little-endian, unsigned short)
        yield pack('<H', rec_size)

    # yield lookup table
    yield pack(f'<{record_count}H', *key_addresses)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Build indexed OKVS structure for search index")
    parser.add_argument("output_game_count_file")
    parser.add_argument("input_hgr_log_file")
    parser.add_argument("input_dhgr_log_file")
    args = parser.parse_args()
    records = [x.strip() for x in sys.stdin.readlines()]
    records = [x.replace('=',',').split(',')
               for x in records
               if x and x[0] not in ('#', '[')]
    for b in build(records, args):
        sys.stdout.buffer.write(b)