Complete: index.py can now build a file just like the "index" external command, including compressing 16 common bytes into 5 bits.

build/index.py

@@ -1,16 +1,11 @@
 #!/usr/bin/python3
 
-# [TODO] extract filename from pathname
-# [TODO] check that we will not overflow the index into the text data
-# [TODO] add the extra index entries if requested
-# [TODO] what is the 4-byte length field in each entry? I don't think it's ever used, but traditionally the index command has set it to something.
-
 # index create indexed.help <index-size-in-bytes>
-# index add    indexed.help <text-file-to-add> [additional-alias-for-this-file]
+# index add    indexed.help <text-file-to-add> [additional-entry-name-aliases-for-this-file]
 
 # Format of an indexed file is:
 #
-# +000   $00 DvxIdx $00
+# +000  $00 DvxIdx $00
 # +008  Offset_to_text  (4 bytes)
 # +012  Offset_to_index (4 bytes)
 # +016  Offset_to_free_index_space (4 bytes)
@@ -18,7 +13,7 @@
 # +052  start of index
 #
 # Index is:
-#   length-pfx'd string, offset(4), length(4)
+#   length-pfx'd string, offset(4), uncompressed-length(4)
 #    :
 #   $00
 #
@@ -27,14 +22,21 @@
 # only 5 bits (%1xxxx), and the remaining ones
 # using 8 bits (%0xxxxxxx).
 
-import sys, struct
+import sys, struct, os.path
+
+SixteenCommonBytes = " etoaisrn\x0dldhpcf"	# Frozen for original format indexed.help files (see index.asm and shell main.asm)
+
+def Compress(ch):
+	fourBits = SixteenCommonBytes.find(chr(ch))
+	if fourBits != -1:
+		return "{:0>5b}".format(16 + fourBits)
+	return "{:0>8b}".format(ch)
 
 verb = sys.argv[1]
 indexFile = sys.argv[2]
 
 if verb == "create":
 	indexSize = int(sys.argv[3])
-	print("We will create with size = " + str(indexSize))
 	offsetToIndex = 52
 	offsetOfText = offsetToIndex + indexSize
 
@@ -47,7 +49,7 @@ if verb == "create":
 		outFile.write(struct.pack('i', 0))	# 8 four-byte integers: reserved
 
 	outFile.seek(offsetOfText - 1)
-	outFile.write(struct.pack('b', 0))
+	outFile.write(struct.pack('B', 0))
 	exit(0)
 
 if verb == "add":
@@ -55,8 +57,14 @@ if verb == "add":
 	addFilePath = sys.argv[3]
 	addFile = open (addFilePath, "r")
 	addLines = addFile.readlines()
+	addFileUncompressedSize = addFile.tell()
 	addFile.close()
 
+	# Read offsetOfText -- the index can't extend that far
+	outFile.seek(8) # read offsetOfText from +008
+	data = outFile.read(4)
+	offsetOfText = int.from_bytes(data, "little")
+
 	# End-of-file offset is where we will put the new chunk of compressed text.
 	outFile.seek(0, 2) # end of file
 	offsetToNewText = outFile.tell()
@@ -65,30 +73,46 @@ if verb == "add":
 	outFile.seek(16)
 	data = outFile.read(4)
 	freeIndexSpace = int.from_bytes(data, "little")
-	print("freeIndexSpace = ", freeIndexSpace)
 
 	# Write new index entry: length-byte, string name, 4-byte offset to start of text, 4-byte length of (compressed?) text
-	entryName = addFilePath		# [TODO] extract just the filename portion
+	entryName = os.path.basename(addFilePath).lower()
 	outFile.seek(freeIndexSpace)
-	outFile.write(struct.pack('b', len(entryName)))
+	outFile.write(struct.pack('B', len(entryName)))
 	outFile.write(bytes(entryName, "utf8"))
 	outFile.write(struct.pack('i', offsetToNewText))
-	outFile.write(struct.pack('i', 0x77777777))	# [TODO] write the length
+	outFile.write(struct.pack('i', addFileUncompressedSize))
 
 	# Optionally write additional index entries for the same text we are adding.
+	for extraName in sys.argv[4:]:
+		outFile.write(struct.pack('B', len(extraName)))
+		outFile.write(bytes(extraName, "utf8"))
+		outFile.write(struct.pack('i', offsetToNewText))
+		outFile.write(struct.pack('i', addFileUncompressedSize))
 
 	# Update freeIndexSpace in file
 	freeIndexSpace = outFile.tell()
+	if freeIndexSpace >= offsetOfText:
+		print("Error: Overflowed the available index space")
+		exit(1)
 	outFile.seek(16)
 	outFile.write(struct.pack('i', freeIndexSpace))
 
 	# Write the new text
 	outFile.seek(offsetToNewText)
+	outBinary = ""
 	for line in addLines:
 		for ch in line.rstrip():
-		    outFile.write(bytes(ch, "utf8"))
-		outFile.write(struct.pack('b', 13)) # carriage return
-	outFile.write(struct.pack('b', 0))
+			outBinary += Compress(ord(ch))
+		outBinary += Compress(13) # carriage return
+	outBinary += Compress(0)
+
+	while len(outBinary) > 0:
+		if len(outBinary) < 8:
+			outBinary = (outBinary + "0000000")[0:8]
+		byte = int(outBinary[0:8], 2)
+		outBinary = outBinary[8:]
+		outFile.write(struct.pack('B', byte))
+
 	outFile.close()
 	exit(0)