build improvements (no binary changes)

This commit is contained in:
4am 2024-06-09 23:40:08 -04:00
parent c656a259c4
commit 1482f1dc69
3 changed files with 125 additions and 121 deletions

View File

@ -84,7 +84,6 @@ extract: preconditions md gamesconf
done; \
rmdir "$$d"; \
done
(for f in build/X.INDEXED/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a -p build/TOTAL.DATA build/X.INDEXED > build/XSINGLE.IDX
index: preconditions md asmfx asmprelaunch asmdemo compress extract
#
@ -103,14 +102,14 @@ index: preconditions md asmfx asmprelaunch asmdemo compress extract
# note: prelaunch must be first in TOTAL.DATA due to a hack in LoadStandardPrelaunch
# note 2: these can not be padded because they are loaded at $0106 and padding would clobber the stack
#
[ -f build/index ] || (bin/buildindexedfile.sh build/TOTAL.DATA build/PRELAUNCH.INDEXED < build/GAMES.SORTED > build/PRELAUNCH.IDX)
[ -f build/index ] || (bin/buildindexedfile.py build/TOTAL.DATA build/PRELAUNCH.INDEXED "" < build/GAMES.SORTED > build/PRELAUNCH.IDX)
#
# precompute indexed files for HGR & DHGR titles
# note: these are not padded because they are all an exact block-multiple anyway
#
[ -f build/index ] || bin/padto.sh build/TOTAL.DATA
[ -f build/index ] || ((for f in res/TITLE.HGR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/TITLE.HGR build/HGR.TITLES.LOG > build/TITLE.IDX)
[ -f build/index ] || ((for f in res/TITLE.DHGR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/TITLE.DHGR build/DHGR.TITLES.LOG > build/DTITLE.IDX)
[ -f build/index ] || ((for f in res/TITLE.HGR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/TITLE.HGR build/HGR.TITLES.LOG > build/TITLE.IDX)
[ -f build/index ] || ((for f in res/TITLE.DHGR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/TITLE.DHGR build/DHGR.TITLES.LOG > build/DTITLE.IDX)
[ -f build/index ] || bin/addfile.sh res/COVER build/TOTAL.DATA > src/index/res.cover.idx.a
[ -f build/index ] || bin/addfile.sh res/TITLE build/TOTAL.DATA > src/index/res.title.idx.a
[ -f build/index ] || bin/addfile.sh res/HELP build/TOTAL.DATA > src/index/res.help.idx.a
@ -118,64 +117,64 @@ index: preconditions md asmfx asmprelaunch asmdemo compress extract
# precompute indexed files for game help
# note: these can be padded because they're loaded into $800 at a time when $800..$1FFF is clobber-able
#
[ -f build/index ] || (bin/buildindexedfile.sh -p -a build/TOTAL.DATA build/GAMEHELP < build/GAMES.SORTED > build/GAMEHELP.IDX)
[ -f build/index ] || (bin/buildindexedfile.py -p -a build/TOTAL.DATA build/GAMEHELP "" < build/GAMES.SORTED > build/GAMEHELP.IDX)
#
# precompute indexed files for slideshows
# note: these can be padded because they're loaded into $800 at a time when $800..$1FFF is clobber-able
#
[ -f build/index ] || $(PARALLEL) '[ $$(echo "{/}" | cut -c-3) = "ACT" ] && bin/buildslideshow.sh -d build/GAMES.CONF < "{}" > "build/SS/{/}" || bin/buildslideshow.sh build/GAMES.CONF < "{}" > "build/SS/{/}"' ::: res/SS/*
[ -f build/index ] || ((for f in build/SS/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -p -a build/TOTAL.DATA build/SS > build/SLIDESHOW.IDX)
[ -f build/index ] || ((for f in build/SS/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -p -a build/TOTAL.DATA build/SS "" > build/SLIDESHOW.IDX)
[ -f build/index ] || $(PARALLEL) 'bin/buildokvs.sh < "{}" > "build/ATTRACT/{/}"' ::: res/ATTRACT/*
[ -f build/index ] || ((for f in build/ATTRACT/[ABCDEFGHIJKLMNOP]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -p -a build/TOTAL.DATA build/ATTRACT > build/MINIATTRACT0.IDX)
[ -f build/index ] || ((for f in build/ATTRACT/[QRSTUVWXYZ]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -p -a build/TOTAL.DATA build/ATTRACT > build/MINIATTRACT1.IDX)
[ -f build/index ] || ((for f in build/ATTRACT/[ABCDEFGHIJKLMNOP]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -p -a build/TOTAL.DATA build/ATTRACT "" > build/MINIATTRACT0.IDX)
[ -f build/index ] || ((for f in build/ATTRACT/[QRSTUVWXYZ]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -p -a build/TOTAL.DATA build/ATTRACT "" > build/MINIATTRACT1.IDX)
#
# precompute indexed files for graphic effects
# note: these can be padded because they're loaded into $6000 at a time when $6000..$BEFF is clobber-able
#
[ -f build/index ] || (bin/buildindexedfile.sh -p -a build/TOTAL.DATA build/FX.INDEXED < res/FX.CONF > build/FX.IDX)
[ -f build/index ] || (bin/buildindexedfile.sh -p -a build/TOTAL.DATA build/FX.INDEXED < res/DFX.CONF > build/DFX.IDX)
[ -f build/index ] || (bin/buildindexedfile.sh -p -a build/TOTAL.DATA build/FX.INDEXED < res/SFX.CONF > build/SFX.IDX)
[ -f build/index ] || ((for f in build/FXCODE/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -p -a build/TOTAL.DATA build/FXCODE > build/FXCODE.IDX)
[ -f build/index ] || (bin/buildindexedfile.py -p -a build/TOTAL.DATA build/FX.INDEXED "" < res/FX.CONF > build/FX.IDX)
[ -f build/index ] || (bin/buildindexedfile.py -p -a build/TOTAL.DATA build/FX.INDEXED "" < res/DFX.CONF > build/DFX.IDX)
[ -f build/index ] || (bin/buildindexedfile.py -p -a build/TOTAL.DATA build/FX.INDEXED "" < res/SFX.CONF > build/SFX.IDX)
[ -f build/index ] || ((for f in build/FXCODE/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -p -a build/TOTAL.DATA build/FXCODE "" > build/FXCODE.IDX)
#
# precompute indexed files for coordinates files loaded by graphic effects
# note: these can not be padded because some of them are loaded into tight spaces near the unclobberable top of main memory
#
[ -f build/index ] || ((for f in build/FXDATA/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA build/FXDATA > build/FXDATA.IDX)
[ -f build/index ] || ((for f in build/FXDATA/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA build/FXDATA "" > build/FXDATA.IDX)
#
# precompute indexed files for HGR & DHGR action screenshots
# note: these can not be padded because they are compressed and the decompressor needs the exact size
#
[ -f build/index ] || ((for f in res/ACTION.HGR/[ABCD]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/ACTION.HGR > build/HGR0.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[EFGH]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/ACTION.HGR > build/HGR1.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[IJKL]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/ACTION.HGR > build/HGR2.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[MNOP]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/ACTION.HGR > build/HGR3.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[QRST]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/ACTION.HGR > build/HGR4.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[UVWX]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/ACTION.HGR > build/HGR5.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[YZ]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/ACTION.HGR > build/HGR6.IDX)
[ -f build/index ] || ((for f in res/ACTION.DHGR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/ACTION.DHGR > build/DHGR.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[ABCD]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/ACTION.HGR "" > build/HGR0.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[EFGH]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/ACTION.HGR "" > build/HGR1.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[IJKL]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/ACTION.HGR "" > build/HGR2.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[MNOP]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/ACTION.HGR "" > build/HGR3.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[QRST]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/ACTION.HGR "" > build/HGR4.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[UVWX]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/ACTION.HGR "" > build/HGR5.IDX)
[ -f build/index ] || ((for f in res/ACTION.HGR/[YZ]*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/ACTION.HGR "" > build/HGR6.IDX)
[ -f build/index ] || ((for f in res/ACTION.DHGR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/ACTION.DHGR "" > build/DHGR.IDX)
#
# precompute indexed files for GR and DGR action screenshots
# note: these can be padded because they are not compressed
#
[ -f build/index ] || ((for f in res/ACTION.GR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a -p build/TOTAL.DATA res/ACTION.GR > build/GR.IDX)
[ -f build/index ] || ((for f in res/ACTION.DGR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a -p build/TOTAL.DATA res/ACTION.DGR > build/DGR.IDX)
[ -f build/index ] || ((for f in res/ACTION.GR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a -p build/TOTAL.DATA res/ACTION.GR "" > build/GR.IDX)
[ -f build/index ] || ((for f in res/ACTION.DGR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a -p build/TOTAL.DATA res/ACTION.DGR "" > build/DGR.IDX)
#
# precompute indexed files for SHR artwork
# note: these can not be padded because they are compressed and the decompressor needs the exact size
#
[ -f build/index ] || ((for f in res/ARTWORK.SHR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA res/ARTWORK.SHR > build/ARTWORK.IDX)
[ -f build/index ] || ((for f in res/ARTWORK.SHR/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA res/ARTWORK.SHR "" > build/ARTWORK.IDX)
#
# precompute indexed files for demo launchers
# note: these can not be padded because some of them are loaded too close to $C000
#
[ -f build/index ] || ((for f in build/DEMO/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a build/TOTAL.DATA build/DEMO > build/DEMO.IDX)
[ -f build/index ] || ((for f in build/DEMO/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a build/TOTAL.DATA build/DEMO "" > build/DEMO.IDX)
[ -f build/index ] || bin/addfile.sh build/DEMO.IDX build/TOTAL.DATA > src/index/demo.idx.a
#
# precompute indexed files for single-load game binaries
# note: these can be padded because they are loaded at a time when all of main memory is clobber-able
#
[ -f build/index ] || ((for f in build/X.INDEXED/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.sh -a -p build/TOTAL.DATA build/X.INDEXED > build/XSINGLE.IDX)
[ -f build/index ] || ((for f in build/X.INDEXED/*; do echo "$$(basename $$f)"; done) | bin/buildindexedfile.py -a -p build/TOTAL.DATA build/X.INDEXED "" > build/XSINGLE.IDX)
[ -f build/index ] || bin/addfile.sh build/XSINGLE.IDX build/TOTAL.DATA > src/index/xsingle.idx.a
#
# create search indexes for each variation of (game-requires-joystick) X (game-requires-128K)

100
bin/buildindexedfile.py Executable file
View File

@ -0,0 +1,100 @@
#!/usr/bin/env python3
# flags
# -a append to data file (default off = truncate)
# -p pad sizes within data file to next block size (default off)
# parameters
# stdin - input containing list of files (e.g. FX.CONF)
# stdout - binary OKVS data structure
# 1 - output filename for data file
# 2 - input directory of files to merge into data file
# 3 - (optional) output filename for log of key,offset,size
import argparse
import os
import os.path
import struct
import sys
kStandardFilename = 'STANDARD'
kFileMode = {False: 'wb', True: 'ab'}
def build(records, args):
if args.append:
output_file_size = os.path.getsize(args.output_file)
else:
output_file_size = 0
standard_offset = standard_size = 0
standard_filename = os.path.join(args.input_directory, kStandardFilename)
with open(args.output_file, kFileMode[args.append]) as output_file_handle:
if os.path.exists(standard_filename):
standard_offset = output_file_size
standard_size = os.path.getsize(standard_filename)
with open(standard_filename, 'rb') as standard_handle:
output_file_handle.write(standard_handle.read())
output_file_size += standard_size
# yield OKVS header (2 x 2 bytes, unsigned int, little-endian)
yield struct.pack('<H', len(records))
yield struct.pack('<H', 0)
for rec in records:
filename, dummy, dummy = rec.partition('=')
key, dummy, addr = filename.partition('#')
filename = os.path.join(args.input_directory, filename)
rec_offset = standard_offset
rec_size = standard_size
# yield record length (1 byte, unsigned byte)
if addr:
# if filename is in the form 'NAME#06ADDR' then create extended index record
yield struct.pack('B', len(key)+9)
# trim '06' so we get just the starting address
addr = addr[2:]
else:
yield struct.pack('B', len(key)+7)
# yield key (Pascal-style string)
yield struct.pack(f'{len(key)+1}p', key.encode('ascii'))
if os.path.exists(filename):
rec_offset = output_file_size
rec_size = os.path.getsize(filename)
output_file_size += rec_size
if args.pad:
# If offset+size does not cross a block boundary, use file's true size.
# Otherwise, round up size to the next block boundary.
# This padding does not get added to the file; it is just an
# optimization to avoid a partial copy on the last block read.
if (rec_offset // 512) != ((rec_offset + rec_size) // 512):
rec_size = ((rec_offset + rec_size + 511) & -512) - rec_offset
with open(filename, 'rb') as input_file_handle:
output_file_handle.write(input_file_handle.read())
# yield record offset (3 bytes, big-endian, unsigned long)
yield struct.pack('>L', rec_offset)[1:]
# yield record size (2 bytes, little-endian, unsigned short)
yield struct.pack('<H', rec_size)
if addr:
# for extended index record, yield load address (2 bytes, little-endian, unsigned short)
yield bytes.fromhex(addr)[::-1]
if args.log_file:
with open(args.log_file, 'a') as log_file_handle:
log_file_handle.write(f'{key},{rec_offset},{rec_size}\n')
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Build indexed OKVS structure with links to data stored in TOTAL.DATA file")
parser.add_argument("output_file")
parser.add_argument("input_directory")
parser.add_argument("log_file")
parser.add_argument("-p", "--pad", action="store_true", default=False, help="pad file sizes to multiples of 512 (default: use exact size)")
parser.add_argument("-a", "--append", action="store_true", default=False, help="append to output file (default: overwrite)")
args = parser.parse_args()
records = [x.strip() for x in sys.stdin.readlines()]
records = [x for x in records if x and x[0] not in ('#', '[')]
for b in build(records, args):
sys.stdout.buffer.write(b)

View File

@ -1,95 +0,0 @@
#!/bin/bash
# flags
# -a append to data file (default off = truncate)
# -p pad sizes within data file to next block size (default off)
# parameters
# stdin - input containing list of files (e.g. FX.CONF)
# stdout - binary OKVS data structure
# 1 - output filename for data file
# 2 - input directory of files to merge into data file
# 3 - (optional) output filename for log of key,offset,size
pad=false
append=false
standardoffset=0
standardsize=0
while getopts ":ap" opt; do
case $opt in
a) append=true
;;
p) pad=true
;;
esac
done
shift $((OPTIND-1))
if [ "$append" = false ]; then
rm -f "$1"
fi
touch "$1"
if [ "${#3}" -ne "0" ]; then
rm -f "$3"
touch "$3"
fi
# if there is a file called "STANDARD" in the input directory, add it now
# because we will reuse it for any files that don't exist
if [ -f "$2"/STANDARD ]; then
standardoffset=$(wc -c < "$1")
standardsize=$(wc -c < "$2/STANDARD")
cat "$2"/STANDARD >> "$1"
fi
# make temp file with list of lines that contain keys
records=$(mktemp)
tr -d "\r" | awk '!/^$|^#/' > "$records"
# make temp assembly source file that represents the binary OKVS data structure
source=$(mktemp)
(echo "*=0" # dummy program counter for assembler
echo "!le16 $(wc -l <"$records"), 0" # OKVS header
while IFS="=" read -r filename dummy; do echo "$filename" | {
IFS="#" read -r key addr
if [ "${#addr}" -ne "0" ]; then # if filename is in the form 'NAME#06ADDR' then create extended index record
addr=$(echo "$addr" | cut -c3-) # trim '06' so we get just the starting address
echo "!byte ${#key}+9" # OKVS record length
else
echo "!byte ${#key}+7" # OKVS record length
fi
echo "!byte ${#key}" # OKVS key length
echo "!text \"$key\"" # OKVS key
if [ ! -e "$2/$filename" ]; then # if file does not exist, use standard offset and size
offset="$standardoffset"
size="$standardsize"
else # otherwise calculate offset and size from file and options
offset=$(wc -c < "$1")
size=$(wc -c < "$2/$filename")
if [ "$pad" = true ]; then
# If offset+size does not cross a block boundary, use file's true size.
# Otherwise, round up size to the next block boundary.
# This padding does not get added to the file; it is just an
# optimization to avoid a partial copy on the last block read.
if [ $(($offset / 512)) -ne $((($offset + $size) / 512)) ]; then
size=$(((($offset + $size + 511) & -512) - $offset))
fi
fi
cat "$2/$filename" >> "$1" # append this file to the end of the merged data file
fi
echo "!be24 $offset"
echo "!le16 $size"
[ "${#addr}" -ne "0" ] && echo '!le16 $'"$addr"
[ "${#3}" -ne "0" ] && echo "$key,$offset,$size" >> "$3"
} done < "$records") > "$source"
# assemble temp source file into binary OKVS data structure, then output that
out=$(mktemp)
acme -o "$out" "$source"
cat "$out"
# clean up
rm "$out"
rm "$source"
rm "$records"