From 55646edc3e2579d00e83c105ff7bbbca28a14637 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 24 Sep 2023 20:56:36 +0200 Subject: [PATCH] added cx16 chunkedfile example --- compiler/test/TestCompilerOnExamples.kt | 1 + examples/cx16/chunkedfile/.gitignore | 5 + examples/cx16/chunkedfile/createmcf.py | 277 ++++++++++++++++++ examples/cx16/chunkedfile/demo.p8 | 64 ++++ examples/cx16/chunkedfile/mcf.p8 | 168 +++++++++++ examples/cx16/chunkedfile/readme.md | 98 +++++++ examples/cx16/chunkedfile/testdata/readme.txt | 3 + 7 files changed, 616 insertions(+) create mode 100644 examples/cx16/chunkedfile/.gitignore create mode 100644 examples/cx16/chunkedfile/createmcf.py create mode 100644 examples/cx16/chunkedfile/demo.p8 create mode 100644 examples/cx16/chunkedfile/mcf.p8 create mode 100644 examples/cx16/chunkedfile/readme.md create mode 100644 examples/cx16/chunkedfile/testdata/readme.txt diff --git a/compiler/test/TestCompilerOnExamples.kt b/compiler/test/TestCompilerOnExamples.kt index 150882f53..591a887fe 100644 --- a/compiler/test/TestCompilerOnExamples.kt +++ b/compiler/test/TestCompilerOnExamples.kt @@ -95,6 +95,7 @@ class TestCompilerOnExamplesCx16: FunSpec({ val onlyCx16 = cartesianProduct( listOf( + "chunkedfile/demo", "vtui/testvtui", "pcmaudio/play-adpcm", "pcmaudio/stream-wav", diff --git a/examples/cx16/chunkedfile/.gitignore b/examples/cx16/chunkedfile/.gitignore new file mode 100644 index 000000000..074df2070 --- /dev/null +++ b/examples/cx16/chunkedfile/.gitignore @@ -0,0 +1,5 @@ +testdata/*TITLESCREEN.* +*.mcf +*.asm +*.prg +*.vice-mon-list diff --git a/examples/cx16/chunkedfile/createmcf.py b/examples/cx16/chunkedfile/createmcf.py new file mode 100644 index 000000000..0cea4a5be --- /dev/null +++ b/examples/cx16/chunkedfile/createmcf.py @@ -0,0 +1,277 @@ +import struct +from typing import Sequence + +# Chunk types: +# user types: 0 - 239 +# reserved: 240 - 249 +CHUNK_DUMMY = 250 +CHUNK_SYSTEMRAM = 251 +CHUNK_VIDEORAM = 252 +CHUNK_PAUSE = 253 +CHUNK_EOF = 254 +CHUNK_IGNORE = 255 + + +class LoadList: + def __init__(self): + self.data = bytearray([CHUNK_IGNORE] * 256) + self.data[0:4] = struct.pack("ccBB", b'L', b'L', 1, 0) + self.index = 4 + + def __eq__(self, other) -> bool: + return isinstance(other, LoadList) and self.data == other.data + + def add_chunk(self, chunktype: int, size: int, bank: int = 0, address: int = 0) -> None: + if chunktype < 0 or chunktype > 255: + raise ValueError("chunktype must be 0 - 255") + if size < 0 or size > 65535: + raise ValueError(f"size must be 0 - 65535 bytes") + if bank < 0 or bank > 31: + raise ValueError("bank must be 0 - 31") + if address < 0 or address > 65535: + raise ValueError("address must be 0 - 65535") + data = struct.pack(" 255: + raise IndexError("too many chunks") + self.data[self.index: self.index + len(data)] = data + self.index += len(data) + + def read(self, data: bytes) -> None: + if len(data) != 256: + raise ValueError("data must be 256 bytes long") + self.data[0:256] = data + self.index = 256 + + def parse(self) -> Sequence[tuple]: + if self.data[0] != ord('L') or self.data[1] != ord('L') or self.data[2] != 1: + raise ValueError("invalid loadlist identifier", self.data[:4]) + index = 4 + chunks = [] + while index < 256: + chunktype = self.data[index] + if chunktype == CHUNK_IGNORE: + index += 1 # pad byte + elif chunktype == CHUNK_EOF: + chunks.append((CHUNK_EOF, 0, 0, 0)) + return chunks + else: + size, bank, address = struct.unpack(" bool: + return self.index <= 4 + + +class MultiChunkFile: + def __init__(self): + self.chunks = [] + self.loadlist = LoadList() + self.datachunks = [] + + def read(self, filename: str) -> None: + num_loadlist = 0 + num_data = 0 + data_total = 0 + file_total = 0 + with open(filename, "rb") as inf: + while True: + data = inf.read(256) + if len(data) == 0: + break + loadlist = LoadList() + loadlist.read(data) + self.chunks.append(loadlist) + num_loadlist += 1 + file_total += len(data) + for chunk in loadlist.parse(): + if chunk[0] == CHUNK_EOF: + break + elif chunk[0] in (CHUNK_DUMMY, CHUNK_SYSTEMRAM, CHUNK_VIDEORAM) or chunk[0] < 240: + data = inf.read(chunk[1]) + self.chunks.append(data) + num_data += 1 + data_total += len(data) + file_total += len(data) + self.validate() + print("Read", filename) + print(f" {num_loadlist} loadlists, {num_data} data chunks") + print(f" total data size: {data_total} (${data_total:x})") + print(f" total file size: {file_total} (${file_total:x})") + + def write(self, filename: str) -> None: + self.flush_ll() + self.validate() + num_loadlist = 0 + num_data = 0 + data_total = 0 + file_total = 0 + with open(filename, "wb") as out: + for chunk in self.chunks: + if isinstance(chunk, LoadList): + num_loadlist += 1 + out.write(chunk.data) + file_total += len(chunk.data) + elif isinstance(chunk, bytes): + num_data += 1 + out.write(chunk) + file_total += len(chunk) + data_total += len(chunk) + else: + raise TypeError("invalid chunk type") + print("Written", filename) + print(f" {num_loadlist} loadlists, {num_data} data chunks") + print(f" total data size: {data_total} (${data_total:x})") + print(f" total file size: {file_total} (${file_total:x})") + + def validate(self) -> None: + if len(self.chunks) < 2: + raise ValueError("must be at least 2 chunks", self.chunks) + chunk_iter = iter(self.chunks) + eof_found = False + while not eof_found: + try: + chunk = next(chunk_iter) + except StopIteration: + raise ValueError("missing EOF chunk") + else: + if isinstance(chunk, LoadList): + for lc in chunk.parse(): + if lc[0] == CHUNK_EOF: + eof_found = True + break + elif lc[0] in (CHUNK_DUMMY, CHUNK_SYSTEMRAM, CHUNK_VIDEORAM) or lc[0] < 240: + size, bank, address = lc[1:] + data = next(chunk_iter) + if isinstance(data, bytes): + if len(data) != size: + raise ValueError("data chunk size mismatch") + else: + raise TypeError("expected data chunk") + else: + raise TypeError("expected LoadList chunk") + try: + next(chunk_iter) + except StopIteration: + pass + else: + raise ValueError("trailing chunks") + + def add_Dummy(self, size: int) -> None: + self.add_chunk(CHUNK_DUMMY, data=bytearray(size)) + + def add_SystemRam(self, bank: int, address: int, data: bytes, chunksize: int=0xfe00) -> None: + if address >= 0xa000 and address < 0xc000: + raise ValueError("use add_BankedRam instead to load chunks into banked ram $a000-$c000") + while data: + if address >= 65536: + raise ValueError("data too large for system ram") + self.add_chunk(CHUNK_SYSTEMRAM, bank, address, data[:chunksize]) + data = data[chunksize:] + address += chunksize + + def add_BankedRam(self, bank: int, address: int, data: bytes, chunksize: int=0x2000) -> None: + if address < 0xa000 or address >= 0xc000: + raise ValueError("use add_SystemRam instead to load chunks into normal system ram") + if chunksize>0x2000: + raise ValueError("chunksize too large for banked ram") + while data: + if address >= 0xc000: + address -= 0xc000 + bank += 1 + if bank >= 32: + raise ValueError("data too large for banked ram") + self.add_chunk(CHUNK_SYSTEMRAM, bank, address, data[:chunksize]) + data = data[chunksize:] + address += chunksize + + def add_VideoRam(self, bank: int, address: int, data: bytes, chunksize: int=0xfe00) -> None: + if bank < 0 or bank > 1: + raise ValueError("bank for videoram must be 0 or 1") + while data: + if address >= 65536: + address -= 65536 + bank += 1 + if bank >= 2: + raise ValueError("data too large for video ram") + self.add_chunk(CHUNK_VIDEORAM, bank, address, data[:chunksize]) + data = data[chunksize:] + address += chunksize + + def add_User(self, chunktype: int, data: bytes) -> None: + if chunktype < 0 or chunktype > 239: + raise ValueError("user chunk type must be 0 - 239") + if len(data) > 65535: + raise ValueError("data too large to fit in a single chunk") + self.add_chunk(chunktype, data=data) + + def add_Pause(self, code: int) -> None: + self.add_chunk_nodata(CHUNK_PAUSE, code) + + def add_EOF(self) -> None: + self.add_chunk_nodata(CHUNK_EOF, 0) + + def add_chunk(self, chunktype: int, bank: int = 0, address: int = 0, data: bytes = b"") -> None: + try: + self.loadlist.add_chunk(chunktype, len(data), bank, address) + except IndexError: + # loadlist is full, flush everything out and create a new one and put it in there. + self.flush_ll() + self.loadlist.add_chunk(chunktype, len(data), bank, address) + if data: + self.datachunks.append(bytes(data)) + + def add_chunk_nodata(self, chunktype: int, code: int = 0) -> None: + try: + self.loadlist.add_chunk(chunktype, code, 0, 0) + except IndexError: + # loadlist is full, flush everything out and create a new one and put it in there. + self.flush_ll() + self.loadlist.add_chunk(chunktype, code, 0, 0) + + def flush_ll(self) -> None: + if not self.loadlist.is_empty(): + self.chunks.append(self.loadlist) + self.chunks.extend(self.datachunks) + self.loadlist = LoadList() + self.datachunks.clear() + + +if __name__ == "__main__": + try: + bitmap1 = open("testdata/ME-TITLESCREEN.BIN", "rb").read() + bitmap2 = open("testdata/DS-TITLESCREEN.BIN", "rb").read() + palette1 = open("testdata/ME-TITLESCREEN.PAL", "rb").read() + palette2 = open("testdata/DS-TITLESCREEN.PAL", "rb").read() + except IOError: + print("""ERROR: cannot load the demo data files. +You'll need to put the titlescreen data files from the 'musicdemo' project into the testdata directory. +The musicdemo is on github: https://github.com/irmen/cx16musicdemo +The four files are the two ME- and the two DS- TITLESCREEN.BIN and .PAL files, and are generated by running the makefile in that project.""") + raise SystemExit + + program = bytearray(3333) + textdata = b"hello this is a demo text.... \x00" + + mcf = MultiChunkFile() + for _ in range(4): + mcf.add_User(42, bytearray(999)) + mcf.add_SystemRam(0, 0x4000, program) + mcf.add_BankedRam(10, 0xa000, textdata) + mcf.add_Pause(444) + for _ in range(4): + mcf.add_User(42, bytearray(999)) + mcf.add_VideoRam(1, 0xfa00, palette1) + mcf.add_VideoRam(0, 0, bitmap1) + mcf.add_Pause(333) + mcf.add_VideoRam(1, 0xfa00, palette2) + mcf.add_VideoRam(0, 0, bitmap2) + mcf.add_Pause(222) + mcf.add_Pause(111) + mcf.add_EOF() + mcf.write("demo.mcf") + print("Verifying file...") + mcf2 = MultiChunkFile() + mcf2.read("demo.mcf") + assert mcf2.chunks == mcf.chunks diff --git a/examples/cx16/chunkedfile/demo.p8 b/examples/cx16/chunkedfile/demo.p8 new file mode 100644 index 000000000..7254baeef --- /dev/null +++ b/examples/cx16/chunkedfile/demo.p8 @@ -0,0 +1,64 @@ +%import textio +%import mcf + +%zeropage basicsafe + +main { + + sub start() { + uword duration + + set_screen_mode() + cbm.SETTIM(0,0,0) + + mcf.set_callbacks(mcf_get_buffer, mcf_process_chunk) ; not needed if the stream has no custom chunk types + if mcf.open("demo.mcf", 8, 2) { + repeat { + mcf.stream() + if_cs { + break ; EOF reached, stop the streaming loop + } else { + ; PAUSE chunk encountered, code is in cx16.r0 + } + } + mcf.close() + } + + duration = cbm.RDTIM16() + cbm.CINT() + txt.print("done. ") + txt.print_uw(duration) + txt.print(" jiffies.\n") + } + + sub set_screen_mode() { + ; 640x400 16 colors + cx16.VERA_DC_VIDEO = (cx16.VERA_DC_VIDEO & %11001111) | %00100000 ; enable only layer 1 + cx16.VERA_DC_HSCALE = 128 + cx16.VERA_DC_VSCALE = 128 + cx16.VERA_CTRL = %00000010 + cx16.VERA_DC_VSTART = 20 + cx16.VERA_DC_VSTOP = 400 /2 -1 + 20 ; clip off screen that overflows vram + cx16.VERA_L1_CONFIG = %00000110 ; 16 colors bitmap mode + cx16.VERA_L1_MAPBASE = 0 + cx16.VERA_L1_TILEBASE = %00000001 ; hires + } + + asmsub mcf_get_buffer(ubyte chunktype @A, uword size @XY) -> ubyte @A, uword @XY, bool @Pc { + %asm {{ + ldx #<$a000 + ldy #>$a000 + lda #10 + clc + rts + }} + } + + asmsub mcf_process_chunk() -> bool @Pc { + ; process the chunk that was loaded in the location returned by the previous call to mcf_get_buffer() + %asm {{ + clc + rts + }} + } +} diff --git a/examples/cx16/chunkedfile/mcf.p8 b/examples/cx16/chunkedfile/mcf.p8 new file mode 100644 index 000000000..305173596 --- /dev/null +++ b/examples/cx16/chunkedfile/mcf.p8 @@ -0,0 +1,168 @@ +%import syslib +%import string + +; Streaming routine for MCF files (multipurpose chunk format): +; 1. call open() +; 2. set callbacks if needed, set_callbacks() +; 3. call stream() in a loop +; 4. call close() if you want to cleanup halfway through for some reason + + +mcf { + uword loadlist_buf = memory("loadlist", 256, 0) + uword @zp loadlist_ptr + bool needs_loadlist + ubyte file_channel + + sub open(str filename, ubyte drive, ubyte channel) -> bool { + cbm.SETNAM(string.length(filename), filename) + cbm.SETLFS(channel, drive, 2) + void cbm.OPEN() + if_cc { + if cbm.READST()==0 { + void cbm.CHKIN(channel) + loadlist_ptr = loadlist_buf + needs_loadlist = true + return true + } + } + close() + return false + } + + sub close() { + cbm.CLRCHN() + cbm.CLOSE(file_channel) + } + + asmsub set_callbacks(uword getbuffer_routine @R0, uword processchunk_routine @R1) { + %asm {{ + lda cx16.r0 + ldy cx16.r0+1 + sta p8_mcf.p8_stream.getbuffer_call+1 + sty p8_mcf.p8_stream.getbuffer_call+2 + lda cx16.r1 + ldy cx16.r1+1 + sta p8_mcf.p8_stream.processchunk_call+1 + sty p8_mcf.p8_stream.processchunk_call+2 + rts + }} + } + + sub stream() { + repeat { + if needs_loadlist { + if not read_loadlist() { + sys.set_carry() + return + } + needs_loadlist = false + } + + while (loadlist_ptr-loadlist_buf)<256 { + when @(loadlist_ptr) { + 255 -> { + ; simply ignore this byte + loadlist_ptr++ + } + 254 -> { + ; End of File + close() + sys.set_carry() + return + } + 253 -> { + ; pause streaming + cx16.r0 = peekw(loadlist_ptr+1) + loadlist_ptr+=6 + sys.clear_carry() + return + } + 252 -> { + ; load into vram + blockload_vram(peekw(loadlist_ptr+1), peek(loadlist_ptr+3), peekw(loadlist_ptr+4)) + loadlist_ptr+=6 + } + 251 -> { + ; load into system ram + blockload_ram(peekw(loadlist_ptr+1), peek(loadlist_ptr+3), peekw(loadlist_ptr+4)) + loadlist_ptr+=6 + } + 250 -> { + ; dummy chunk + blockload_dummy(peekw(loadlist_ptr+1)) + loadlist_ptr+=6 + } + else -> { + ; custom chunk + uword @shared chunksize = peekw(loadlist_ptr+1) + %asm {{ + lda (p8_mcf.p8_loadlist_ptr) + ldx p8_chunksize + ldy p8_chunksize+1 +getbuffer_call jsr $ffff ; modified + bcc + + rts ; fail - exit as if EOF ++ sta cx16.r1L + stx cx16.r0 + sty cx16.r0+1 + }} + blockload_ram(chunksize, cx16.r1L, cx16.r0) + loadlist_ptr += 6 + %asm {{ +processchunk_call jsr $ffff ; modified + bcc + + rts ++ + }} + } + } + } + needs_loadlist = true + } + } + + sub read_loadlist() -> bool { + blockload_ram(256, cx16.getrambank(), loadlist_buf) + if loadlist_buf[0]!=sc:'L' or loadlist_buf[1]!=sc:'L' or loadlist_buf[2]!=1 + return false ; header error + loadlist_ptr = loadlist_buf+4 + } + + sub blockload_vram(uword size, ubyte bank, uword address) { + cx16.VERA_CTRL = 0 + cx16.VERA_ADDR_L = lsb(address) + cx16.VERA_ADDR_M = msb(address) + cx16.VERA_ADDR_H = bank | %00010000 ; enable vera auto increment + while size { + size -= readblock(size, &cx16.VERA_DATA0, true) + } + } + + sub blockload_dummy(uword size) { + ubyte buffer + while size { + size -= readblock(size, &buffer, true) + } + } + + sub blockload_ram(uword size, ubyte bank, uword address) { + ubyte orig_ram_bank = cx16.getrambank() + cx16.rambank(bank) + cx16.r3 = address + while size { + cx16.r2 = readblock(size, cx16.r3, false) + size -= cx16.r2 + cx16.r3 += cx16.r2 + } + cx16.rambank(orig_ram_bank) + } + + sub readblock(uword size, uword address, bool dontAdvance) -> uword { + if msb(size)>=2 + return cx16.macptr(0, address, dontAdvance) ; read 512 bytes + if msb(size) + return cx16.macptr(255, address, dontAdvance) ; read 255 bytes + return cx16.macptr(lsb(size), address, dontAdvance) ; read remaining number of bytes + } +} \ No newline at end of file diff --git a/examples/cx16/chunkedfile/readme.md b/examples/cx16/chunkedfile/readme.md new file mode 100644 index 000000000..17f13559e --- /dev/null +++ b/examples/cx16/chunkedfile/readme.md @@ -0,0 +1,98 @@ +# Multipurpose Chunked File + +## Goals + +- meant for the Commander X16 system, depends on working MACPTR kernal call for fast loading. +- single file, possibly megabytes in size, that contains various kinds of data to be loaded or streamed into different locations. +- simple file format with few special cases. +- no random access, only sequential reading/streaming. +- simple user code that doesn't have to bother with the I/O at all. +- a few chunk typs that can be handled automatically to load data into system ram, banked ram, or video ram. +- custom chunk types for flexibility or extensibility. + +Theoretical optimal chunk size is 512 bytes but actual size may be different. (In practice there seems to be no significant speed impact) + +MCF files are meant to be be created using a tool on PC, and only being read on the X16. +A Python tool is provided to create a demo MCF file. +A proof of concept Prog8 library module and example program is provided to consume that demo MCF file on the X16. + + +## File Format + +This is the MCF file format: + +| Chunk | size (bytes) | +|------------|--------------| +| LoadList | 256 | +| Data | 1 - 65535 | +| Data | 1 - 65535 | +| ... | | +| LoadList | 256 | +| Data | 1 - 65535 | +| Data | 1 - 65535 | +| ... | | + +and so on. +There is no limit to the number of chunks and the size of the file, as long as it fits on the disk. + + +### LoadList chunk + +This chunk is a list of what kinds of chunks occur in the file after it. +It starts with a small identification header: + +| data | meaning | +|---------|---------------------| +| 2 bytes | 'LL' (76, 76) | +| byte | version (1 for now) | +| byte | reserved (0) | + +Then a sequence of 1 or more chunk specs (6 bytes each), as long as it still fits in 256 bytes: + +| data | meaning | +|----------------------|--------------------------------------------| +| byte | chunk type | +| word (little-endian) | chunk size | +| byte | bank number (used for some chunk types) | +| word (little-endian) | memory address (used for some chunk types) | + +Total 6 bytes per occurrence. Any remaining unused bytes in the 256 bytes LoadList chunk are to be padded with byte 255. +If there are more chunks in the file than fit in a single loadlist, we simply add another loadlist block and continue there. +(The file only ends once an End Of File chunk type is encountered in a loadlist.) + + +### Chunk types + +| chunk type | meaning | +|--------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0 - 239 | custom chunk types. See below. | +| 240 - 249 | reserved for future system chunk types. | +| 250 | dummy chunk: read a chunk of the specified number of bytes but don't do anything with it. Useful to realign the file I/O on disk block size. | +| 251 | system ram load: use banknumber + address to set the RAM bank and load address and loads the chunk there, then continue streaming. | +| 252 | video ram load: use banknumber + address to set the Vera VRAM bank (hi byte) and load address (mid+lo byte) and loads the chunk into video ram there, then continue streaming. | +| 253 | pause streaming. Returns from stream routine with pause status: Carry=clear. And reg.r0=size. until perhaps the program calls the stream routine again to resume. | +| 254 | end of file. Closes the file and stops streaming: returns from stream routine with exit status: Carry=set. | +| 255 | ignore this byte. Used to pad out the loadlist chunk to 256 bytes. | + + +### Custom chunk types (0 - 239) + +When such a custom chunk type is encountered, a user routine is called to get the load address for the chunk data, +then the chunk is loaded into that buffer, and finally a user routine is called to process the chunk data. +The size can be zero, so that the chunk type acts like a simple notification flag for the main program to do something. + +The first routine has the following signature: + +**get_buffer()**: + Arguments: reg.A = chunk type, reg.XY = chunksize. + Returns: Carry flag=success (set = fail, clear = ok), ram bank in reg.A, memory address in reg.XY. + +The second routine has the following signature: + +**process_chunk()**: + Arguments: none (save them from the get_buffer call if needed). + Returns: Carry flag=success (set = fail, clear = ok). + +These routines are provided to the streaming routine as callback addresses (ram bank number + address to call). +If any of these routines returns Carry set (error status) the streaming routine halts, otherwise it keeps on going. +The streaming continues until a End of File chunk type is encountered in the loadlist. diff --git a/examples/cx16/chunkedfile/testdata/readme.txt b/examples/cx16/chunkedfile/testdata/readme.txt new file mode 100644 index 000000000..acb04cc22 --- /dev/null +++ b/examples/cx16/chunkedfile/testdata/readme.txt @@ -0,0 +1,3 @@ +You'll need to put the titlescreen data files from the 'musicdemo' project into this directory. +The musicdemo is on github: https://github.com/irmen/cx16musicdemo +The four files are the two ME- and the two DS- TITLESCREEN.BIN and .PAL files, and are generated by running the makefile in that project.