apple1-worple/packer.py
2022-05-14 10:48:07 -05:00

103 lines
3.0 KiB
Python

# Read in file of five-letter words and encode them into two groups
# of three-byte packed binary.
#
# Each letter is converted to a five-bit value, 0-25.
# The bits are put together into a 25 bits, which would normally take
# up four bytes of storage. Since bit 0 of the MSB (the right-most bit
# of our 25 bits) can only be 0 or 1, we'll just split the data
# into two groups and only store the least-significant 24 bits.
#
# In other words, if the first letter is A-P (0-15), we can store that
# first letter in only four bits. Remaining letters are five bits each
# regardless, for a total of 24 bits (three bytes).
# If the first letter is P-Z (15-25), we'll still store it in four bits,
# but in separate list that we'll keep in a specific section of
# memory. We'll just need to be aware that words from that part of the
# list will need to have that bit put back on (i.e., have that first
# letter shifted up the alphabet by 16 letters)
import random
INFILE = 'words.txt'
OUTFILE_0 = 'words_0.bin'
OUTFILE_1 = 'words_1.bin'
def pack(word):
# Pack a five-letter word into three bytes.
# Return binary bytes along with an overflow flag (1 or 0)
word = word.strip().upper()
if len(word) != 5:
raise ValueError(f'"{word}" is not five letters long')
b = 0
for i in range(5):
# Convert letter to 0-25 integer
c = ord(word[i]) - 65
if (c < 0) or (c > 25):
raise ValueError('Character out of range A-Z in word ' + word)
# Scoot current contents of b over by five bits,
# then add new character
b = b * 32
b += c
# Did we "overflow" into bit 24?
if b & 0x1000000:
# Mask off just the leftmost three bytes
b &= 0xFFFFFF
overflow = 1
else:
overflow = 0
# Return as a three-byte bytes object
return [b.to_bytes(3, 'big'), overflow]
def head_and_tail(list, size=6):
print('Head: ' + str(list[0:size]))
print('Tail: ' + str(list[-size:]))
return
##############################################################################
# Read in list
print('\nReading ' + INFILE)
with open(INFILE, 'r') as infile:
words = infile.readlines()
print('Words read: ' + str(len(words)))
head_and_tail(words)
print()
# Shuffle them up
print('Shuffling...')
random.seed(24601)
random.shuffle(words)
head_and_tail(words)
print()
# Pack into bytes (this also uppercases and strips whitespace/returns)
print('Packing words...')
packed_list = [[],[]]
for word in words:
packed = pack(word)
packed_list[packed[1]].append(packed[0])
print('Words with first bit = 0: ' + str(len(packed_list[0])))
print('Words with first bit = 1: ' + str(len(packed_list[1])))
print()
# Save as two separate list files
print('Saving word lists...')
with open(OUTFILE_0, 'wb') as outfile:
for data in packed_list[0]:
outfile.write(data)
with open(OUTFILE_1, 'wb') as outfile:
for data in packed_list[1]:
outfile.write(data)
print('Done\n')