added cx16 chunkedfile example

This commit is contained in:
Irmen de Jong 2023-09-24 20:56:36 +02:00
parent 8d177beb78
commit 55646edc3e
7 changed files with 616 additions and 0 deletions

View File

@ -95,6 +95,7 @@ class TestCompilerOnExamplesCx16: FunSpec({
val onlyCx16 = cartesianProduct(
listOf(
"chunkedfile/demo",
"vtui/testvtui",
"pcmaudio/play-adpcm",
"pcmaudio/stream-wav",

5
examples/cx16/chunkedfile/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
testdata/*TITLESCREEN.*
*.mcf
*.asm
*.prg
*.vice-mon-list

View File

@ -0,0 +1,277 @@
import struct
from typing import Sequence
# Chunk types:
# user types: 0 - 239
# reserved: 240 - 249
CHUNK_DUMMY = 250
CHUNK_SYSTEMRAM = 251
CHUNK_VIDEORAM = 252
CHUNK_PAUSE = 253
CHUNK_EOF = 254
CHUNK_IGNORE = 255
class LoadList:
def __init__(self):
self.data = bytearray([CHUNK_IGNORE] * 256)
self.data[0:4] = struct.pack("ccBB", b'L', b'L', 1, 0)
self.index = 4
def __eq__(self, other) -> bool:
return isinstance(other, LoadList) and self.data == other.data
def add_chunk(self, chunktype: int, size: int, bank: int = 0, address: int = 0) -> None:
if chunktype < 0 or chunktype > 255:
raise ValueError("chunktype must be 0 - 255")
if size < 0 or size > 65535:
raise ValueError(f"size must be 0 - 65535 bytes")
if bank < 0 or bank > 31:
raise ValueError("bank must be 0 - 31")
if address < 0 or address > 65535:
raise ValueError("address must be 0 - 65535")
data = struct.pack("<BHBH", chunktype, size, bank, address)
if self.index + len(data) > 255:
raise IndexError("too many chunks")
self.data[self.index: self.index + len(data)] = data
self.index += len(data)
def read(self, data: bytes) -> None:
if len(data) != 256:
raise ValueError("data must be 256 bytes long")
self.data[0:256] = data
self.index = 256
def parse(self) -> Sequence[tuple]:
if self.data[0] != ord('L') or self.data[1] != ord('L') or self.data[2] != 1:
raise ValueError("invalid loadlist identifier", self.data[:4])
index = 4
chunks = []
while index < 256:
chunktype = self.data[index]
if chunktype == CHUNK_IGNORE:
index += 1 # pad byte
elif chunktype == CHUNK_EOF:
chunks.append((CHUNK_EOF, 0, 0, 0))
return chunks
else:
size, bank, address = struct.unpack("<HBH", self.data[index + 1:index + 6])
chunks.append((chunktype, size, bank, address))
index += 6
return chunks
def is_empty(self) -> bool:
return self.index <= 4
class MultiChunkFile:
def __init__(self):
self.chunks = []
self.loadlist = LoadList()
self.datachunks = []
def read(self, filename: str) -> None:
num_loadlist = 0
num_data = 0
data_total = 0
file_total = 0
with open(filename, "rb") as inf:
while True:
data = inf.read(256)
if len(data) == 0:
break
loadlist = LoadList()
loadlist.read(data)
self.chunks.append(loadlist)
num_loadlist += 1
file_total += len(data)
for chunk in loadlist.parse():
if chunk[0] == CHUNK_EOF:
break
elif chunk[0] in (CHUNK_DUMMY, CHUNK_SYSTEMRAM, CHUNK_VIDEORAM) or chunk[0] < 240:
data = inf.read(chunk[1])
self.chunks.append(data)
num_data += 1
data_total += len(data)
file_total += len(data)
self.validate()
print("Read", filename)
print(f" {num_loadlist} loadlists, {num_data} data chunks")
print(f" total data size: {data_total} (${data_total:x})")
print(f" total file size: {file_total} (${file_total:x})")
def write(self, filename: str) -> None:
self.flush_ll()
self.validate()
num_loadlist = 0
num_data = 0
data_total = 0
file_total = 0
with open(filename, "wb") as out:
for chunk in self.chunks:
if isinstance(chunk, LoadList):
num_loadlist += 1
out.write(chunk.data)
file_total += len(chunk.data)
elif isinstance(chunk, bytes):
num_data += 1
out.write(chunk)
file_total += len(chunk)
data_total += len(chunk)
else:
raise TypeError("invalid chunk type")
print("Written", filename)
print(f" {num_loadlist} loadlists, {num_data} data chunks")
print(f" total data size: {data_total} (${data_total:x})")
print(f" total file size: {file_total} (${file_total:x})")
def validate(self) -> None:
if len(self.chunks) < 2:
raise ValueError("must be at least 2 chunks", self.chunks)
chunk_iter = iter(self.chunks)
eof_found = False
while not eof_found:
try:
chunk = next(chunk_iter)
except StopIteration:
raise ValueError("missing EOF chunk")
else:
if isinstance(chunk, LoadList):
for lc in chunk.parse():
if lc[0] == CHUNK_EOF:
eof_found = True
break
elif lc[0] in (CHUNK_DUMMY, CHUNK_SYSTEMRAM, CHUNK_VIDEORAM) or lc[0] < 240:
size, bank, address = lc[1:]
data = next(chunk_iter)
if isinstance(data, bytes):
if len(data) != size:
raise ValueError("data chunk size mismatch")
else:
raise TypeError("expected data chunk")
else:
raise TypeError("expected LoadList chunk")
try:
next(chunk_iter)
except StopIteration:
pass
else:
raise ValueError("trailing chunks")
def add_Dummy(self, size: int) -> None:
self.add_chunk(CHUNK_DUMMY, data=bytearray(size))
def add_SystemRam(self, bank: int, address: int, data: bytes, chunksize: int=0xfe00) -> None:
if address >= 0xa000 and address < 0xc000:
raise ValueError("use add_BankedRam instead to load chunks into banked ram $a000-$c000")
while data:
if address >= 65536:
raise ValueError("data too large for system ram")
self.add_chunk(CHUNK_SYSTEMRAM, bank, address, data[:chunksize])
data = data[chunksize:]
address += chunksize
def add_BankedRam(self, bank: int, address: int, data: bytes, chunksize: int=0x2000) -> None:
if address < 0xa000 or address >= 0xc000:
raise ValueError("use add_SystemRam instead to load chunks into normal system ram")
if chunksize>0x2000:
raise ValueError("chunksize too large for banked ram")
while data:
if address >= 0xc000:
address -= 0xc000
bank += 1
if bank >= 32:
raise ValueError("data too large for banked ram")
self.add_chunk(CHUNK_SYSTEMRAM, bank, address, data[:chunksize])
data = data[chunksize:]
address += chunksize
def add_VideoRam(self, bank: int, address: int, data: bytes, chunksize: int=0xfe00) -> None:
if bank < 0 or bank > 1:
raise ValueError("bank for videoram must be 0 or 1")
while data:
if address >= 65536:
address -= 65536
bank += 1
if bank >= 2:
raise ValueError("data too large for video ram")
self.add_chunk(CHUNK_VIDEORAM, bank, address, data[:chunksize])
data = data[chunksize:]
address += chunksize
def add_User(self, chunktype: int, data: bytes) -> None:
if chunktype < 0 or chunktype > 239:
raise ValueError("user chunk type must be 0 - 239")
if len(data) > 65535:
raise ValueError("data too large to fit in a single chunk")
self.add_chunk(chunktype, data=data)
def add_Pause(self, code: int) -> None:
self.add_chunk_nodata(CHUNK_PAUSE, code)
def add_EOF(self) -> None:
self.add_chunk_nodata(CHUNK_EOF, 0)
def add_chunk(self, chunktype: int, bank: int = 0, address: int = 0, data: bytes = b"") -> None:
try:
self.loadlist.add_chunk(chunktype, len(data), bank, address)
except IndexError:
# loadlist is full, flush everything out and create a new one and put it in there.
self.flush_ll()
self.loadlist.add_chunk(chunktype, len(data), bank, address)
if data:
self.datachunks.append(bytes(data))
def add_chunk_nodata(self, chunktype: int, code: int = 0) -> None:
try:
self.loadlist.add_chunk(chunktype, code, 0, 0)
except IndexError:
# loadlist is full, flush everything out and create a new one and put it in there.
self.flush_ll()
self.loadlist.add_chunk(chunktype, code, 0, 0)
def flush_ll(self) -> None:
if not self.loadlist.is_empty():
self.chunks.append(self.loadlist)
self.chunks.extend(self.datachunks)
self.loadlist = LoadList()
self.datachunks.clear()
if __name__ == "__main__":
try:
bitmap1 = open("testdata/ME-TITLESCREEN.BIN", "rb").read()
bitmap2 = open("testdata/DS-TITLESCREEN.BIN", "rb").read()
palette1 = open("testdata/ME-TITLESCREEN.PAL", "rb").read()
palette2 = open("testdata/DS-TITLESCREEN.PAL", "rb").read()
except IOError:
print("""ERROR: cannot load the demo data files.
You'll need to put the titlescreen data files from the 'musicdemo' project into the testdata directory.
The musicdemo is on github: https://github.com/irmen/cx16musicdemo
The four files are the two ME- and the two DS- TITLESCREEN.BIN and .PAL files, and are generated by running the makefile in that project.""")
raise SystemExit
program = bytearray(3333)
textdata = b"hello this is a demo text.... \x00"
mcf = MultiChunkFile()
for _ in range(4):
mcf.add_User(42, bytearray(999))
mcf.add_SystemRam(0, 0x4000, program)
mcf.add_BankedRam(10, 0xa000, textdata)
mcf.add_Pause(444)
for _ in range(4):
mcf.add_User(42, bytearray(999))
mcf.add_VideoRam(1, 0xfa00, palette1)
mcf.add_VideoRam(0, 0, bitmap1)
mcf.add_Pause(333)
mcf.add_VideoRam(1, 0xfa00, palette2)
mcf.add_VideoRam(0, 0, bitmap2)
mcf.add_Pause(222)
mcf.add_Pause(111)
mcf.add_EOF()
mcf.write("demo.mcf")
print("Verifying file...")
mcf2 = MultiChunkFile()
mcf2.read("demo.mcf")
assert mcf2.chunks == mcf.chunks

View File

@ -0,0 +1,64 @@
%import textio
%import mcf
%zeropage basicsafe
main {
sub start() {
uword duration
set_screen_mode()
cbm.SETTIM(0,0,0)
mcf.set_callbacks(mcf_get_buffer, mcf_process_chunk) ; not needed if the stream has no custom chunk types
if mcf.open("demo.mcf", 8, 2) {
repeat {
mcf.stream()
if_cs {
break ; EOF reached, stop the streaming loop
} else {
; PAUSE chunk encountered, code is in cx16.r0
}
}
mcf.close()
}
duration = cbm.RDTIM16()
cbm.CINT()
txt.print("done. ")
txt.print_uw(duration)
txt.print(" jiffies.\n")
}
sub set_screen_mode() {
; 640x400 16 colors
cx16.VERA_DC_VIDEO = (cx16.VERA_DC_VIDEO & %11001111) | %00100000 ; enable only layer 1
cx16.VERA_DC_HSCALE = 128
cx16.VERA_DC_VSCALE = 128
cx16.VERA_CTRL = %00000010
cx16.VERA_DC_VSTART = 20
cx16.VERA_DC_VSTOP = 400 /2 -1 + 20 ; clip off screen that overflows vram
cx16.VERA_L1_CONFIG = %00000110 ; 16 colors bitmap mode
cx16.VERA_L1_MAPBASE = 0
cx16.VERA_L1_TILEBASE = %00000001 ; hires
}
asmsub mcf_get_buffer(ubyte chunktype @A, uword size @XY) -> ubyte @A, uword @XY, bool @Pc {
%asm {{
ldx #<$a000
ldy #>$a000
lda #10
clc
rts
}}
}
asmsub mcf_process_chunk() -> bool @Pc {
; process the chunk that was loaded in the location returned by the previous call to mcf_get_buffer()
%asm {{
clc
rts
}}
}
}

View File

@ -0,0 +1,168 @@
%import syslib
%import string
; Streaming routine for MCF files (multipurpose chunk format):
; 1. call open()
; 2. set callbacks if needed, set_callbacks()
; 3. call stream() in a loop
; 4. call close() if you want to cleanup halfway through for some reason
mcf {
uword loadlist_buf = memory("loadlist", 256, 0)
uword @zp loadlist_ptr
bool needs_loadlist
ubyte file_channel
sub open(str filename, ubyte drive, ubyte channel) -> bool {
cbm.SETNAM(string.length(filename), filename)
cbm.SETLFS(channel, drive, 2)
void cbm.OPEN()
if_cc {
if cbm.READST()==0 {
void cbm.CHKIN(channel)
loadlist_ptr = loadlist_buf
needs_loadlist = true
return true
}
}
close()
return false
}
sub close() {
cbm.CLRCHN()
cbm.CLOSE(file_channel)
}
asmsub set_callbacks(uword getbuffer_routine @R0, uword processchunk_routine @R1) {
%asm {{
lda cx16.r0
ldy cx16.r0+1
sta p8_mcf.p8_stream.getbuffer_call+1
sty p8_mcf.p8_stream.getbuffer_call+2
lda cx16.r1
ldy cx16.r1+1
sta p8_mcf.p8_stream.processchunk_call+1
sty p8_mcf.p8_stream.processchunk_call+2
rts
}}
}
sub stream() {
repeat {
if needs_loadlist {
if not read_loadlist() {
sys.set_carry()
return
}
needs_loadlist = false
}
while (loadlist_ptr-loadlist_buf)<256 {
when @(loadlist_ptr) {
255 -> {
; simply ignore this byte
loadlist_ptr++
}
254 -> {
; End of File
close()
sys.set_carry()
return
}
253 -> {
; pause streaming
cx16.r0 = peekw(loadlist_ptr+1)
loadlist_ptr+=6
sys.clear_carry()
return
}
252 -> {
; load into vram
blockload_vram(peekw(loadlist_ptr+1), peek(loadlist_ptr+3), peekw(loadlist_ptr+4))
loadlist_ptr+=6
}
251 -> {
; load into system ram
blockload_ram(peekw(loadlist_ptr+1), peek(loadlist_ptr+3), peekw(loadlist_ptr+4))
loadlist_ptr+=6
}
250 -> {
; dummy chunk
blockload_dummy(peekw(loadlist_ptr+1))
loadlist_ptr+=6
}
else -> {
; custom chunk
uword @shared chunksize = peekw(loadlist_ptr+1)
%asm {{
lda (p8_mcf.p8_loadlist_ptr)
ldx p8_chunksize
ldy p8_chunksize+1
getbuffer_call jsr $ffff ; modified
bcc +
rts ; fail - exit as if EOF
+ sta cx16.r1L
stx cx16.r0
sty cx16.r0+1
}}
blockload_ram(chunksize, cx16.r1L, cx16.r0)
loadlist_ptr += 6
%asm {{
processchunk_call jsr $ffff ; modified
bcc +
rts
+
}}
}
}
}
needs_loadlist = true
}
}
sub read_loadlist() -> bool {
blockload_ram(256, cx16.getrambank(), loadlist_buf)
if loadlist_buf[0]!=sc:'L' or loadlist_buf[1]!=sc:'L' or loadlist_buf[2]!=1
return false ; header error
loadlist_ptr = loadlist_buf+4
}
sub blockload_vram(uword size, ubyte bank, uword address) {
cx16.VERA_CTRL = 0
cx16.VERA_ADDR_L = lsb(address)
cx16.VERA_ADDR_M = msb(address)
cx16.VERA_ADDR_H = bank | %00010000 ; enable vera auto increment
while size {
size -= readblock(size, &cx16.VERA_DATA0, true)
}
}
sub blockload_dummy(uword size) {
ubyte buffer
while size {
size -= readblock(size, &buffer, true)
}
}
sub blockload_ram(uword size, ubyte bank, uword address) {
ubyte orig_ram_bank = cx16.getrambank()
cx16.rambank(bank)
cx16.r3 = address
while size {
cx16.r2 = readblock(size, cx16.r3, false)
size -= cx16.r2
cx16.r3 += cx16.r2
}
cx16.rambank(orig_ram_bank)
}
sub readblock(uword size, uword address, bool dontAdvance) -> uword {
if msb(size)>=2
return cx16.macptr(0, address, dontAdvance) ; read 512 bytes
if msb(size)
return cx16.macptr(255, address, dontAdvance) ; read 255 bytes
return cx16.macptr(lsb(size), address, dontAdvance) ; read remaining number of bytes
}
}

View File

@ -0,0 +1,98 @@
# Multipurpose Chunked File
## Goals
- meant for the Commander X16 system, depends on working MACPTR kernal call for fast loading.
- single file, possibly megabytes in size, that contains various kinds of data to be loaded or streamed into different locations.
- simple file format with few special cases.
- no random access, only sequential reading/streaming.
- simple user code that doesn't have to bother with the I/O at all.
- a few chunk typs that can be handled automatically to load data into system ram, banked ram, or video ram.
- custom chunk types for flexibility or extensibility.
Theoretical optimal chunk size is 512 bytes but actual size may be different. (In practice there seems to be no significant speed impact)
MCF files are meant to be be created using a tool on PC, and only being read on the X16.
A Python tool is provided to create a demo MCF file.
A proof of concept Prog8 library module and example program is provided to consume that demo MCF file on the X16.
## File Format
This is the MCF file format:
| Chunk | size (bytes) |
|------------|--------------|
| LoadList | 256 |
| Data | 1 - 65535 |
| Data | 1 - 65535 |
| ... | |
| LoadList | 256 |
| Data | 1 - 65535 |
| Data | 1 - 65535 |
| ... | |
and so on.
There is no limit to the number of chunks and the size of the file, as long as it fits on the disk.
### LoadList chunk
This chunk is a list of what kinds of chunks occur in the file after it.
It starts with a small identification header:
| data | meaning |
|---------|---------------------|
| 2 bytes | 'LL' (76, 76) |
| byte | version (1 for now) |
| byte | reserved (0) |
Then a sequence of 1 or more chunk specs (6 bytes each), as long as it still fits in 256 bytes:
| data | meaning |
|----------------------|--------------------------------------------|
| byte | chunk type |
| word (little-endian) | chunk size |
| byte | bank number (used for some chunk types) |
| word (little-endian) | memory address (used for some chunk types) |
Total 6 bytes per occurrence. Any remaining unused bytes in the 256 bytes LoadList chunk are to be padded with byte 255.
If there are more chunks in the file than fit in a single loadlist, we simply add another loadlist block and continue there.
(The file only ends once an End Of File chunk type is encountered in a loadlist.)
### Chunk types
| chunk type | meaning |
|--------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 0 - 239 | custom chunk types. See below. |
| 240 - 249 | reserved for future system chunk types. |
| 250 | dummy chunk: read a chunk of the specified number of bytes but don't do anything with it. Useful to realign the file I/O on disk block size. |
| 251 | system ram load: use banknumber + address to set the RAM bank and load address and loads the chunk there, then continue streaming. |
| 252 | video ram load: use banknumber + address to set the Vera VRAM bank (hi byte) and load address (mid+lo byte) and loads the chunk into video ram there, then continue streaming. |
| 253 | pause streaming. Returns from stream routine with pause status: Carry=clear. And reg.r0=size. until perhaps the program calls the stream routine again to resume. |
| 254 | end of file. Closes the file and stops streaming: returns from stream routine with exit status: Carry=set. |
| 255 | ignore this byte. Used to pad out the loadlist chunk to 256 bytes. |
### Custom chunk types (0 - 239)
When such a custom chunk type is encountered, a user routine is called to get the load address for the chunk data,
then the chunk is loaded into that buffer, and finally a user routine is called to process the chunk data.
The size can be zero, so that the chunk type acts like a simple notification flag for the main program to do something.
The first routine has the following signature:
**get_buffer()**:
Arguments: reg.A = chunk type, reg.XY = chunksize.
Returns: Carry flag=success (set = fail, clear = ok), ram bank in reg.A, memory address in reg.XY.
The second routine has the following signature:
**process_chunk()**:
Arguments: none (save them from the get_buffer call if needed).
Returns: Carry flag=success (set = fail, clear = ok).
These routines are provided to the streaming routine as callback addresses (ram bank number + address to call).
If any of these routines returns Carry set (error status) the streaming routine halts, otherwise it keeps on going.
The streaming continues until a End of File chunk type is encountered in the loadlist.

View File

@ -0,0 +1,3 @@
You'll need to put the titlescreen data files from the 'musicdemo' project into this directory.
The musicdemo is on github: https://github.com/irmen/cx16musicdemo
The four files are the two ME- and the two DS- TITLESCREEN.BIN and .PAL files, and are generated by running the makefile in that project.