Refactor the world

main.py

@@ -3,7 +3,9 @@ import skvideo.datasets
 from PIL import Image
 import numpy as np
 
+import opcodes
 import screen
+import video
 
 CYCLES = 1024 * 1024
 MAX_OUT = 20 * 1024
@@ -45,113 +47,121 @@ APPLE_FPS = 10
 # stores=2222, content changes=339, page changes=88
 
 # Optimized new
-# Fullness = 1.384560, cycles = 90738/104857 budget
+# Fullness = 1.384560, cycle_counter = 90738/104857 budget
 # stores=1872, content changes=15, page changes=352
 # Frame 0, 2606 bytes, similarity = 0.849219
-# Fullness = 1.452588, cycles = 110009/104857 budget
+# Fullness = 1.452588, cycle_counter = 110009/104857 budget
 # stores=2163, content changes=28, page changes=472
 # Frame 3, 3163 bytes, similarity = 0.924256
-# Fullness = 1.577072, cycles = 113843/104857 budget
+# Fullness = 1.577072, cycle_counter = 113843/104857 budget
 # stores=2062, content changes=30, page changes=577
 # Frame 6, 3276 bytes, similarity = 0.939918
-# Fullness = 1.597466, cycles = 106213/104857 budget
+# Fullness = 1.597466, cycle_counter = 106213/104857 budget
 # stores=1899, content changes=29, page changes=550
 # Frame 9, 3057 bytes, similarity = 0.928274
-# Fullness = 1.615001, cycles = 106008/104857 budget
+# Fullness = 1.615001, cycle_counter = 106008/104857 budget
 # stores=1875, content changes=27, page changes=561
 # Frame 12, 3051 bytes, similarity = 0.933854
-# Fullness = 1.639691, cycles = 106460/104857 budget
+# Fullness = 1.639691, cycle_counter = 106460/104857 budget
 # stores=1855, content changes=30, page changes=575
 # Frame 15, 3065 bytes, similarity = 0.929725
-# Fullness = 1.635406, cycles = 104583/104857 budget
+# Fullness = 1.635406, cycle_counter = 104583/104857 budget
 # stores=1827, content changes=30, page changes=562
 
 # TSP solver
-# Fullness = 1.336189, cycles = 87568/104857 budget
+# Fullness = 1.336189, cycle_counter = 87568/104857 budget
 # stores=1872, content changes=320, page changes=32
 # Frame 0, 2576 bytes, similarity = 0.849219
-# Fullness = 1.386065, cycles = 108771/104857 budget
+# Fullness = 1.386065, cycle_counter = 108771/104857 budget
 # stores=2242, content changes=452, page changes=33
 # Frame 3, 3212 bytes, similarity = 0.927604
-# Fullness = 1.482284, cycles = 112136/104857 budget
+# Fullness = 1.482284, cycle_counter = 112136/104857 budget
 # stores=2161, content changes=552, page changes=33
 # Frame 6, 3331 bytes, similarity = 0.943415
-# Fullness = 1.501014, cycles = 106182/104857 budget
+# Fullness = 1.501014, cycle_counter = 106182/104857 budget
 # stores=2021, content changes=535, page changes=33
 # Frame 9, 3157 bytes, similarity = 0.934263
-# Fullness = 1.523818, cycles = 106450/104857 budget
+# Fullness = 1.523818, cycle_counter = 106450/104857 budget
 # stores=1995, content changes=554, page changes=33
 # Frame 12, 3169 bytes, similarity = 0.939844
-# Fullness = 1.543029, cycles = 106179/104857 budget
+# Fullness = 1.543029, cycle_counter = 106179/104857 budget
 # stores=1966, content changes=566, page changes=33
 # Frame 15, 3164 bytes, similarity = 0.935231
-# Fullness = 1.538659, cycles = 104560/104857 budget
+# Fullness = 1.538659, cycle_counter = 104560/104857 budget
 # stores=1941, content changes=554, page changes=33
 
 # page first
-# Fullness = 1.366463, cycles = 89552/104857 budget
+# Fullness = 1.366463, cycle_counter = 89552/104857 budget
 # stores=1872, content changes=352, page changes=32
 # Frame 0, 2640 bytes, similarity = 0.849219
-# Fullness = 1.413155, cycles = 108440/104857 budget
+# Fullness = 1.413155, cycle_counter = 108440/104857 budget
 # stores=2192, content changes=476, page changes=32
 # Frame 3, 3208 bytes, similarity = 0.925744
-# Fullness = 1.516888, cycles = 112554/104857 budget
+# Fullness = 1.516888, cycle_counter = 112554/104857 budget
 # stores=2120, content changes=583, page changes=32
 # Frame 6, 3350 bytes, similarity = 0.942187
-# Fullness = 1.535086, cycles = 106115/104857 budget
+# Fullness = 1.535086, cycle_counter = 106115/104857 budget
 # stores=1975, content changes=561, page changes=32
 # Frame 9, 3161 bytes, similarity = 0.932106
-# Fullness = 1.553913, cycles = 106143/104857 budget
+# Fullness = 1.553913, cycle_counter = 106143/104857 budget
 # stores=1951, content changes=575, page changes=32
 # Frame 12, 3165 bytes, similarity = 0.937835
-# Fullness = 1.571548, cycles = 106047/104857 budget
+# Fullness = 1.571548, cycle_counter = 106047/104857 budget
 # stores=1927, content changes=587, page changes=32
 # Frame 15, 3165 bytes, similarity = 0.933259
-# Fullness = 1.572792, cycles = 104940/104857 budget
+# Fullness = 1.572792, cycle_counter = 104940/104857 budget
 # stores=1906, content changes=581, page changes=32
 
 def main():
-    s = screen.Screen()
-
-    decoder = screen.Screen()
+    s = video.Video()
 
     videogen = skvideo.io.vreader("CoffeeCup-H264-75.mov")
     with open("out.bin", "wb") as out:
         bytes_out = 0
 
-        # Estimated opcode overhead, i.e. ratio of extra cycles from opcodes
+        # Estimated opcode overhead, i.e. ratio of extra cycle_counter from
+        # opcodes
         fullness = 1.6
 
+        screen_cls = screen.HGRBitmap
+
+        # Assert that the opcode stream reconstructs the same screen
+        ds = video.Video()
+        decoder = opcodes.Decoder(ds.state)
+
         for idx, frame in enumerate(videogen):
             if idx % (VIDEO_FPS // APPLE_FPS):
                 continue
+
             im = Image.fromarray(frame)
-            im = im.resize((screen.Frame.XMAX, screen.Frame.YMAX))
+            im = im.resize((screen_cls.XMAX, screen_cls.YMAX))
             im = im.convert("1")
             im = np.array(im)
             # im.show()
 
-            f = screen.Frame(im)
+            f = screen_cls(im)
+
             cycle_budget = int(CYCLES / APPLE_FPS)
             stream = bytes(s.update(f, cycle_budget, fullness))
 
-            fullness *= s.cycles / cycle_budget
-            print("Fullness = %f, cycles = %d/%d budget" % (
-                fullness, s.cycles, cycle_budget))
+            #print(" ".join("%02x(%02d)" % (b, b) for b in stream))
+
+            fullness *= s.cycle_counter.cycles / cycle_budget
+            print("Fullness = %f, cycle_counter = %d/%d budget" % (
+                fullness, s.cycle_counter.cycles, cycle_budget))
 
-            # Assert that the opcode stream reconstructs the same screen
             (num_content_stores, num_content_changes,
-             num_page_changes, num_rle_bytes) = decoder.from_stream(iter(
-                stream))
-            assert np.array_equal(decoder.screen, s.screen)
+             num_page_changes, num_rle_bytes) = decoder.decode_stream(
+                iter(stream))
+            assert np.array_equal(ds.screen.bytemap, s.screen.bytemap), (
+                    ds.screen.bytemap ^ s.screen.bytemap)
             print("stores=%d, content changes=%d, page changes=%d, "
                   "rle_bytes=%d" % (
                       num_content_stores, num_content_changes,
                       num_page_changes, num_rle_bytes))
 
-            # print(" ".join("%02x(%02d)" % (b, b) for b in stream))
             # assert that the screen decodes to the original bitmap
-            bm = s.to_bitmap()
+            bm = screen_cls.from_bytemap(s.screen).bitmap
 
             #           print(np.array(im)[0:5,0:5])
             #            print(bm[0:5,0:5])
@@ -171,7 +181,7 @@ def main():
                 break
 
             print("Frame %d, %d bytes, similarity = %f" % (
-                idx, len(stream), s.similarity(im, bm)))
+                idx, len(stream), screen.bitmap_similarity(im, bm)))
             out.write(stream)
 
         out.write(bytes(s.done()))

memory_map.py

@@ -0,0 +1,54 @@
+from typing import Tuple
+
+import screen
+
+
+def y_to_base_addr(y: int, page: int = 0) -> int:
+    """Maps y coordinate to base address on given screen page"""
+    a = y // 64
+    d = y - 64 * a
+    b = d // 8
+    c = d - 8 * b
+
+    addr = 8192 * (page + 1) + 1024 * c + 128 * b + 40 * a
+    return addr
+
+
+class MemoryMap:
+    """Memory map representing screen memory."""
+
+    # TODO: support DHGR
+
+    Y_TO_BASE_ADDR = [
+        [y_to_base_addr(y, screen_page) for y in range(192)]
+        for screen_page in (0, 1)
+    ]
+
+    ADDR_TO_COORDS = {}
+    for p in range(2):
+        for y in range(192):
+            for x in range(40):
+                a = Y_TO_BASE_ADDR[p][y] + x
+                ADDR_TO_COORDS[a] = (p, y, x)
+
+    def __init__(self, screen_page: int, bytemap: screen.Bytemap):
+        self.screen_page = screen_page  # type: int
+        self.bytemap = bytemap
+
+    def to_page_offset(self, x_byte: int, y: int) -> Tuple[int, int]:
+        y_base = self.Y_TO_BASE_ADDR[self.screen_page][y]
+        page = y_base >> 8
+
+        # print("y=%d -> page=%02x" % (y, page))
+        offset = y_base - (page << 8) + x_byte
+        return page, offset
+
+    def write(self, addr: int, val: int) -> None:
+        """Updates screen image to set 0xaddr ^= val"""
+        try:
+            _, y, x = self.ADDR_TO_COORDS[addr]
+        except KeyError:
+            # TODO: filter out screen holes
+            # print("Attempt to write to invalid offset %04x" % addr)
+            return
+        self.bytemap.bytemap[y][x] = val

opcodes.py

@@ -0,0 +1,194 @@
+import enum
+from typing import Iterator, Tuple
+
+import memory_map
+
+
+class CycleCounter:
+    def __init__(self):
+        self.cycles = 0  # type:int
+
+    def tick(self, cycles: int) -> None:
+        self.cycles += cycles
+
+    def reset(self) -> None:
+        self.cycles = 0
+
+
+class State:
+    """Represents virtual machine state."""
+
+    def __init__(self, cycle_counter: CycleCounter,
+                 memmap: memory_map.MemoryMap):
+        self.page = 0x20
+        self.content = 0x7f
+
+        self.memmap = memmap
+        self.cycle_counter = cycle_counter
+
+    def emit(self, opcode: "Opcode") -> Iterator[int]:
+        cmd = opcode.emit_command()
+        if cmd:
+            yield from cmd
+        data = opcode.emit_data()
+        if data:
+            yield from data
+
+        # Update changes in memory map, if any
+        opcode.apply(self)
+
+        # Tick 6502 CPU
+        self.cycle_counter.tick(opcode.cycles)
+
+
+class OpcodeCommand(enum.Enum):
+    OFFSET = 0x00
+    SET_CONTENT = 0xfb  # set new data byte to write
+    SET_PAGE = 0xfc
+    RLE = 0xfd
+    TICK = 0xfe  # tick speaker
+    TERMINATE = 0xff
+
+
+class Opcode:
+    COMMAND = None  # type: OpcodeCommand
+    _CYCLES = None  # type: int
+
+    @property
+    def cycles(self) -> int:
+        return self._CYCLES
+
+    def emit_command(self) -> Iterator[int]:
+        yield self.COMMAND.value
+
+    def emit_data(self) -> Iterator[int]:
+        return
+
+    def apply(self, state: State):
+        pass
+
+
+class Offset(Opcode):
+    COMMAND = OpcodeCommand.OFFSET
+    _CYCLES = 36
+
+    def __init__(self, offset: int):
+        if offset < 0 or offset >255:
+            raise ValueError("Invalid offset: %d" % offset)
+
+        self.offset = offset
+
+    def emit_command(self):
+        return
+
+    def emit_data(self):
+        yield self.offset
+
+    def apply(self, state):
+        state.memmap.write(state.page << 8 | self.offset, state.content)
+
+
+class SetPage(Opcode):
+    COMMAND = OpcodeCommand.SET_PAGE
+    _CYCLES = 73
+
+    def __init__(self, page:int):
+        self.page = page
+
+    def emit_data(self):
+        yield self.page
+
+    def apply(self, state: State):
+        state.page = self.page
+
+
+class SetContent(Opcode):
+    COMMAND = OpcodeCommand.SET_CONTENT
+    _CYCLES = 62
+
+    def __init__(self, content: int):
+        self.content = content
+
+    def emit_data(self):
+        yield self.content
+
+    def apply(self, state: State):
+        state.content = self.content
+
+
+class RLE(Opcode):
+    COMMAND = OpcodeCommand.RLE
+
+    def __init__(self, start_offset: int, run_length: int):
+        self.start_offset = start_offset
+        self.run_length = run_length
+
+    def emit_data(self):
+        yield self.start_offset
+        yield self.run_length
+
+    @property
+    def cycles(self):
+        return 98 + 9 * self.run_length
+
+    def apply(self, state):
+        for i in range(self.run_length):
+            state.memmap.write(
+                state.page << 8 | ((self.start_offset + i) & 0xff),
+                state.content
+            )
+
+
+class Tick(Opcode):
+    COMMAND = OpcodeCommand.TICK
+    _CYCLES = 50
+
+
+class Terminate(Opcode):
+    COMMAND = OpcodeCommand.TERMINATE
+    _CYCLES = 50
+
+
+class Decoder:
+    def __init__(self, state: State):
+        self.state = state  # type: State
+
+    def decode_stream(self, stream: Iterator[int]) -> Tuple[int, int, int, int]:
+        """Replay an opcode stream to build a screen image."""
+        num_content_changes = 0
+        num_page_changes = 0
+        num_content_stores = 0
+        num_rle_bytes = 0
+
+        terminate = False
+        for b in stream:
+            if b == OpcodeCommand.SET_CONTENT.value:
+                content = next(stream)
+                op = SetContent(content)
+                num_content_changes += 1
+            elif b == OpcodeCommand.SET_PAGE.value:
+                page = next(stream)
+                op = SetPage(page)
+                num_page_changes += 1
+            elif b == OpcodeCommand.RLE.value:
+                offset = next(stream)
+                run_length = next(stream)
+                num_rle_bytes += run_length
+                op = RLE(offset, run_length)
+            elif b == OpcodeCommand.TICK.value:
+                op = Tick()
+            elif b == OpcodeCommand.TERMINATE.value:
+                op = Terminate()
+                terminate = True
+            else:
+                op = Offset(b)
+                num_content_stores += 1
+
+            op.apply(self.state)
+            if terminate:
+                break
+
+        return (
+            num_content_stores, num_content_changes, num_page_changes,
+            num_rle_bytes
+        )

scheduler.py

@@ -0,0 +1,125 @@
+"""Opcode schedulers."""
+
+from typing import Iterator
+
+import opcodes
+
+
+class OpcodeScheduler:
+    def schedule(self, changes) -> Iterator[opcodes.Opcode]:
+        raise NotImplementedError
+
+
+class HeuristicPageFirstScheduler(OpcodeScheduler):
+    """Group by page first then content byte."""
+
+    def schedule(self, changes):
+        data = {}
+        for ch in changes:
+            xor_weight, page, offset, content, run_length = ch
+            data.setdefault(page, {}).setdefault(content, set()).add(
+                (run_length, offset))
+
+        for page, content_offsets in data.items():
+            yield opcodes.SetPage(page)
+            for content, offsets in content_offsets.items():
+                yield opcodes.SetContent(content)
+
+                # print("page %d content %d offsets %s" % (page, content,
+                #                                         offsets))
+                for (run_length, offset) in sorted(offsets, reverse=True):
+                    if run_length > 1:
+                        # print("Offset %d run length %d" % (
+                        #     offset, run_length))
+                        yield opcodes.RLE(offset, run_length)
+                    else:
+                        yield opcodes.Offset(offset)
+
+#
+# def _tsp_opcode_scheduler(self, changes):
+#     # Build distance matrix for pairs of changes based on number of
+#     # opcodes it would cost for opcodes to emit target change given source
+#
+#     dist = np.zeros(shape=(len(changes), len(changes)), dtype=np.int)
+#     for i1, ch1 in enumerate(changes):
+#         _, page1, _, content1 = ch1
+#         for i2, ch2 in enumerate(changes):
+#             if ch1 == ch2:
+#                 continue
+#             _, page2, _, content2 = ch2
+#
+#             cost = self.CYCLES[0]  # Emit the target content byte
+#             if page1 != page2:
+#                 cost += self.CYCLES[OpcodeCommand.SET_PAGE]
+#             if content1 != content2:
+#                 cost += self.CYCLES[OpcodeCommand.SET_CONTENT]
+#
+#             dist[i1][i2] = cost
+#             dist[i2][i1] = cost
+#
+#     def create_distance_callback(dist_matrix):
+#         # Create a callback to calculate distances between cities.
+#
+#         def distance_callback(from_node, to_node):
+#             return int(dist_matrix[from_node][to_node])
+#
+#         return distance_callback
+#
+#     routing = pywrapcp.RoutingModel(len(changes), 1, 0)
+#     search_parameters = pywrapcp.RoutingModel.DefaultSearchParameters()
+#     # Create the distance callback.
+#     dist_callback = create_distance_callback(dist)
+#     routing.SetArcCostEvaluatorOfAllVehicles(dist_callback)
+#
+#     assignment = routing.SolveWithParameters(search_parameters)
+#     if assignment:
+#         # Solution distance.
+#         print("Total cycle_counter: " + str(assignment.ObjectiveValue()))
+#         # Display the solution.
+#         # Only one route here; otherwise iterate from 0 to
+#         # routing.vehicles() - 1
+#         route_number = 0
+#         index = routing.Start(
+#             route_number)  # Index of the variable for the starting node.
+#         page = 0x20
+#         content = 0x7f
+#         # TODO: I think this will end by visiting the origin node which
+#         #  is not what we want
+#         while not routing.IsEnd(index):
+#             _, new_page, offset, new_content = changes[index]
+#
+#             if new_page != page:
+#                 page = new_page
+#                 yield self._emit(OpcodeCommand.SET_PAGE)
+#                 yield page
+#
+#             if new_content != content:
+#                 content = new_content
+#                 yield self._emit(OpcodeCommand.SET_CONTENT)
+#                 yield content
+#
+#             self._write(page << 8 | offset, content)
+#             yield self._emit(offset)
+#
+#             index = assignment.Value(routing.NextVar(index))
+#     else:
+#         raise ValueError('No solution found.')
+#
+# def _heuristic_opcode_scheduler(self, changes):
+#     # Heuristic: group by content byte first then page
+#     data = {}
+#     for ch in changes:
+#         xor_weight, page, offset, content = ch
+#         data.setdefault(content, {}).setdefault(page, set()).add(offset)
+#
+#     for content, page_offsets in data.items():
+#         yield self._emit(OpcodeCommand.SET_CONTENT)
+#         yield content
+#         for page, offsets in page_offsets.items():
+#             yield self._emit(OpcodeCommand.SET_PAGE)
+#             yield page
+#
+#             for offset in offsets:
+#                 self._write(page << 8 | offset, content)
+#                 yield self._emit(offset)
+#

screen.py

@@ -1,100 +1,34 @@
 """Screen module represents Apple II video display."""
 
-from collections import defaultdict
-import functools
-import enum
-from typing import List, Set, Iterator, Union, Tuple
-
-from ortools.constraint_solver import pywrapcp
-from ortools.constraint_solver import routing_enums_pb2
+# from ortools.constraint_solver import pywrapcp
+# from ortools.constraint_solver import routing_enums_pb2
 
 import numpy as np
 
 
-@functools.lru_cache(None)
-def hamming_weight(n: int) -> int:
-    """Compute hamming weight of 8-bit int"""
-    n = (n & 0x55) + ((n & 0xAA) >> 1)
-    n = (n & 0x33) + ((n & 0xCC) >> 2)
-    n = (n & 0x0F) + ((n & 0xF0) >> 4)
-    return n
+def bitmap_similarity(a1: np.array, a2: np.array) -> float:
+    """Measure bitwise % similarity between two bitmap arrays"""
+    bits_different = np.asscalar(np.sum(np.logical_xor(a1, a2)))
+
+    return 1 - (bits_different / (np.shape(a1)[0] * np.shape(a1)[1]))
 
 
-def y_to_base_addr(y: int, page: int = 0) -> int:
-    """Maps y coordinate to base address on given screen page"""
-    a = y // 64
-    d = y - 64 * a
-    b = d // 8
-    c = d - 8 * b
+class Bytemap:
+    """Bitmap array with horizontal pixels packed into bytes."""
 
-    addr = 8192 * (page + 1) + 1024 * c + 128 * b + 40 * a
-    return addr
+    def __init__(self, bitmap: np.array):
+        self.ymax = bitmap.shape[0]  # type: int
+        self.xmax = bitmap.shape[1]  # type: int
+        if self.xmax % 7 != 0:
+            raise ValueError(
+                "Bitmap x dimension not divisible by 7: %d" % self.xmax)
 
+        self._unpacked_bitmap = bitmap
 
-# TODO: fill out other byte opcodes
-class Opcode(enum.Enum):
-    SET_CONTENT = 0xfb  # set new data byte to write
-    SET_PAGE = 0xfc
-    RLE = 0xfd
-    TICK = 0xfe  # tick speaker
-    END_FRAME = 0xff
+        self.bytemap = self._pack()  # type: np.array
 
-
-class Frame:
-    """Bitmapped screen frame."""
-
-    XMAX = 280  # 140  # double-wide pixels to not worry about colour
-    # effects
-    YMAX = 192
-
-    def __init__(self, bitmap: np.array = None):
-        if bitmap is None:
-            self.bitmap = np.zeros((self.YMAX, self.XMAX), dtype=bool)
-        else:
-            self.bitmap = bitmap
-
-    def randomize(self):
-        self.bitmap = np.random.randint(
-            2, size=(self.YMAX, self.XMAX), dtype=bool)
-
-
-class Screen:
-    """Apple II screen memory map encoding a bitmapped frame."""
-
-    Y_TO_BASE_ADDR = [
-        [y_to_base_addr(y, page) for y in range(192)] for page in (0, 1)
-    ]
-
-    ADDR_TO_COORDS = {}
-    for p in range(2):
-        for y in range(192):
-            for x in range(40):
-                a = Y_TO_BASE_ADDR[p][y] + x
-                ADDR_TO_COORDS[a] = (p, y, x)
-
-    CYCLES = defaultdict(lambda: 36)  # fast-path cycle count
-    CYCLES.update({
-        Opcode.SET_CONTENT: 62,
-        Opcode.SET_PAGE: 73,
-        Opcode.RLE: 98,  # + 9 * N
-        Opcode.TICK: 50,
-        Opcode.END_FRAME: 50
-    })
-
-    def __init__(self, page: int = 0):
-        self.screen = self._encode(Frame().bitmap)  # initialize empty
-        self.page = page
-        self.cycles = 0
-
-    @staticmethod
-    def _encode(bitmap: np.array) -> np.array:
-        """Encode bitmapped screen as apple II memory map.
-
-        Rows are y-coordinates, Columns are byte-packed x-values
-        """
-
-        # Double each pixel horizontally
-        pixels = bitmap # np.repeat(bitmap, 2, axis=1)
+    def _pack(self) -> np.array:
+        pixels = self._unpacked_bitmap
 
         # Insert zero column after every 7
         for i in range(pixels.shape[1] // 7 - 1, -1, -1):
@@ -104,299 +38,9 @@ class Screen:
         # invert this
         return np.flip(np.packbits(np.flip(pixels, axis=1), axis=1), axis=1)
 
-    def update(self, frame: Frame,
-               cycle_budget: int, fullness: float) -> Iterator[int]:
-        """Update to match content of frame within provided budget.
-
-        Emits encoded byte stream for rendering the image.
-
-        The byte stream consists of offsets against a selected page (e.g. $20xx)
-        at which to write a selected content byte.  Those selections are
-        controlled by special opcodes emitted to the stream
-
-        Opcodes:
-          SET_CONTENT - new byte to write to screen contents
-          SET_PAGE - set new page to offset against (e.g. $20xx)
-          TICK - tick the speaker
-          DONE - terminate the video decoding
-
-        In order to "make room" for these opcodes we make use of the fact that
-        each page has 2 sets of 8-byte "screen holes", at page offsets
-        0x78-0x7f and 0xf8-0xff.  Currently we only use the latter range as
-        this allows for efficient matching in the critical path of the decoder.
-
-        We group by offsets from page boundary (cf some other more
-        optimal starting point) because STA (..),y has 1 extra cycle if
-        crossing the page boundary.  Though maybe this would be worthwhile if
-        it optimizes the bytestream.
-        """
-
-        self.cycles = 0
-        # Target screen memory map for new frame
-        target = self._encode(frame.bitmap)
-
-        # Estimate number of opcodes that will end up fitting in the cycle
-        # budget.
-        est_opcodes = int(cycle_budget / fullness / self.CYCLES[0])
-
-        # Sort by highest xor weight and take the estimated number of change
-        # operations
-        changes = list(
-            sorted(self.index_changes(self.screen, target), reverse=True)
-        )[:est_opcodes]
-
-        for b in self._heuristic_page_first_opcode_scheduler(changes):
-            yield b
-
-    def index_changes(self, source: np.array,
-                      target: np.array) -> Set[Tuple[int, int, int, int, int]]:
-        """Transform encoded screen to sequence of change tuples.
-
-        Change tuple is (xor_weight, page, offset, content)
-        """
-
-        # Compute difference from current frame
-        deltas = np.bitwise_xor(self.screen, target)
-        deltas = np.ma.masked_array(deltas, np.logical_not(deltas))
-
-        changes = set()
-        # TODO: don't use 256 bytes if XMAX is smaller, or we may compute RLE
-        # over the full page!
-        memmap = defaultdict(lambda: [(0, 0, 0)] * 256)
-
-        it = np.nditer(target, flags=['multi_index'])
-        while not it.finished:
-            y, x_byte = it.multi_index
-
-
-            y_base = self.Y_TO_BASE_ADDR[self.page][y]
-            page = y_base >> 8
-
-            # print("y=%d -> page=%02x" % (y, page))
-            offset = y_base - (page << 8) + x_byte
-
-            src_content = source[y][x_byte]
-            target_content = np.asscalar(it[0])
-
-            bits_different = hamming_weight(src_content ^ target_content)
-
-            memmap[page][offset] = (bits_different, src_content, target_content)
-            it.iternext()
-
-        for page, offsets in memmap.items():
-            cur_content = None
-            run_length = 0
-            maybe_run = []
-            for offset, data in enumerate(offsets):
-                bits_different, src_content, target_content = data
-
-                # TODO: allowing odd bit errors introduces colour error
-                if maybe_run and hamming_weight(
-                        cur_content ^ target_content) > 2:
-                    # End of run
-
-                    # Decide if it's worth emitting as a run vs single stores
-
-                    # Number of changes in run for which >0 bits differ
-                    num_changes = len([c for c in maybe_run if c[0]])
-                    run_cost = self.CYCLES[Opcode.RLE] + run_length * 9
-                    single_cost = self.CYCLES[0] * num_changes
-                    #print("Run of %d cheaper than %d singles" % (
-                    #    run_length, num_changes))
-
-                    # TODO: don't allow too much error to accumulate
-
-                    if run_cost < single_cost:
-                        # Compute median bit value over run
-                        median_bits = np.median(
-                            np.vstack(
-                                np.unpackbits(
-                                    np.array(r[3], dtype=np.uint8)
-                                )
-                                for r in maybe_run
-                            ), axis=0
-                        ) > 0.5
-
-                        typical_content = np.asscalar(np.packbits(median_bits))
-
-                        total_xor = sum(ch[0] for ch in maybe_run)
-                        start_offset = maybe_run[0][2]
-
-                        change = (total_xor, page, start_offset,
-                                  typical_content, run_length)
-                        # print("Found run of %d * %2x at %2x:%2x" % (
-                        #    run_length, cur_content, page, offset - run_length)
-                        #      )
-                        #print(maybe_run)
-                        #print("change =", change)
-                        changes.add(change)
-                    else:
-                        changes.update(ch for ch in maybe_run if ch[0])
-                    maybe_run = []
-                    run_length = 0
-                    cur_content = target_content
-
-                if cur_content is None:
-                    cur_content = target_content
-
-                run_length += 1
-#                if bits_different != 0:
-#                    # Only accumulate bytes for which src != target content
-                maybe_run.append(
-                    (bits_different, page, offset, target_content, 1))
-
-        return changes
-
-    def _heuristic_page_first_opcode_scheduler(self, changes):
-        # Heuristic: group by page first then content byte
-        data = {}
-        for ch in changes:
-            xor_weight, page, offset, content, run_length = ch
-            data.setdefault(page, {}).setdefault(content, set()).add(
-                (run_length, offset))
-
-        for page, content_offsets in data.items():
-            for b in self._emit(Opcode.SET_PAGE, page):
-                yield b
-            for content, offsets in content_offsets.items():
-                for b in self._emit(Opcode.SET_CONTENT, content):
-                    yield b
-
-                # print("page %d content %d offsets %s" % (page, content,
-                #                                         offsets))
-                for (run_length, offset) in sorted(offsets, reverse=True):
-                    if run_length > 1:
-                        # print("Offset %d run length %d" % (offset, run_length))
-                        for b in self._emit(Opcode.RLE, offset, run_length):
-                            yield b
-                        for i in range(run_length):
-                            self._write((page << 8 | offset) + i, content)
-                    else:
-                        for b in self._emit(offset):
-                            yield b
-                        self._write(page << 8 | offset, content)
-
-    def _tsp_opcode_scheduler(self, changes):
-        # Build distance matrix for pairs of changes based on number of
-        # opcodes it would cost for opcodes to emit target change given source
-
-        dist = np.zeros(shape=(len(changes), len(changes)), dtype=np.int)
-        for i1, ch1 in enumerate(changes):
-            _, page1, _, content1 = ch1
-            for i2, ch2 in enumerate(changes):
-                if ch1 == ch2:
-                    continue
-                _, page2, _, content2 = ch2
-
-                cost = self.CYCLES[0]  # Emit the target content byte
-                if page1 != page2:
-                    cost += self.CYCLES[Opcode.SET_PAGE]
-                if content1 != content2:
-                    cost += self.CYCLES[Opcode.SET_CONTENT]
-
-                dist[i1][i2] = cost
-                dist[i2][i1] = cost
-
-        def create_distance_callback(dist_matrix):
-            # Create a callback to calculate distances between cities.
-
-            def distance_callback(from_node, to_node):
-                return int(dist_matrix[from_node][to_node])
-
-            return distance_callback
-
-        routing = pywrapcp.RoutingModel(len(changes), 1, 0)
-        search_parameters = pywrapcp.RoutingModel.DefaultSearchParameters()
-        # Create the distance callback.
-        dist_callback = create_distance_callback(dist)
-        routing.SetArcCostEvaluatorOfAllVehicles(dist_callback)
-
-        assignment = routing.SolveWithParameters(search_parameters)
-        if assignment:
-            # Solution distance.
-            print("Total cycles: " + str(assignment.ObjectiveValue()))
-            # Display the solution.
-            # Only one route here; otherwise iterate from 0 to routing.vehicles() - 1
-            route_number = 0
-            index = routing.Start(
-                route_number)  # Index of the variable for the starting node.
-            page = 0x20
-            content = 0x7f
-            # TODO: I think this will end by visiting the origin node which
-            #  is not what we want
-            while not routing.IsEnd(index):
-                _, new_page, offset, new_content = changes[index]
-
-                if new_page != page:
-                    page = new_page
-                    yield self._emit(Opcode.SET_PAGE)
-                    yield page
-
-                if new_content != content:
-                    content = new_content
-                    yield self._emit(Opcode.SET_CONTENT)
-                    yield content
-
-                self._write(page << 8 | offset, content)
-                yield self._emit(offset)
-
-                index = assignment.Value(routing.NextVar(index))
-        else:
-            raise ValueError('No solution found.')
-
-    def _heuristic_opcode_scheduler(self, changes):
-        # Heuristic: group by content byte first then page
-        data = {}
-        for ch in changes:
-            xor_weight, page, offset, content = ch
-            data.setdefault(content, {}).setdefault(page, set()).add(offset)
-
-        for content, page_offsets in data.items():
-            yield self._emit(Opcode.SET_CONTENT)
-            yield content
-            for page, offsets in page_offsets.items():
-                yield self._emit(Opcode.SET_PAGE)
-                yield page
-
-                for offset in offsets:
-                    self._write(page << 8 | offset, content)
-                    yield self._emit(offset)
-
-    def _emit(self, opcode: Union[Opcode, int], *data) -> List[int]:
-        if opcode == Opcode.RLE:
-            run_length = data[1]
-            self.cycles += 9 * run_length
-        self.cycles += self.CYCLES[opcode]
-
-        opcode_byte = opcode.value if opcode in Opcode else opcode
-        return [opcode_byte] + list(data)
-
-    @staticmethod
-    def similarity(a1: np.array, a2: np.array) -> float:
-        """Measure bitwise % similarity between two arrays"""
-        bits_different = np.asscalar(np.sum(np.logical_xor(a1, a2)))
-
-        return 1 - (bits_different / (np.shape(a1)[0] * np.shape(a1)[1]))
-
-    def done(self) -> Iterator[int]:
-        """Terminate opcode stream."""
-
-        for b in self._emit(Opcode.END_FRAME):
-            yield b
-
-    def _write(self, addr: int, val: int) -> None:
-        """Updates screen image to set 0xaddr ^= val"""
-        try:
-            _, y, x = self.ADDR_TO_COORDS[addr]
-        except KeyError:
-            # TODO: filter out screen holes
-            # print("Attempt to write to invalid offset %04x" % addr)
-            return
-        self.screen[y][x] = val
-
-    def to_bitmap(self) -> np.array:
+    def unpack(self) -> np.array:
         """Convert packed screen representation to bitmap."""
-        bm = np.unpackbits(self.screen, axis=1)
+        bm = np.unpackbits(self.bytemap, axis=1)
         bm = np.delete(bm, np.arange(0, bm.shape[1], 8), axis=1)
 
         # Need to flip each 7-bit sequence
@@ -406,44 +50,55 @@ class Screen:
                 reorder_cols.append(j)
         bm = bm[:, reorder_cols]
 
-        # Undouble pixels
-        # return np.array(np.delete(bm, np.arange(0, bm.shape[1], 2), axis=1),
-        #                dtype=np.bool)
         return np.array(bm, dtype=np.bool)
 
-    def from_stream(self, stream: Iterator[int]) -> Tuple[int, int, int]:
-        """Replay an opcode stream to build a screen image."""
-        page = 0x20
-        content = 0x7f
-        num_content_changes = 0
-        num_page_changes = 0
-        num_content_stores = 0
-        num_rle_bytes = 0
-        for b in stream:
-            if b == Opcode.SET_CONTENT.value:
-                content = next(stream)
-                num_content_changes += 1
-                continue
-            elif b == Opcode.SET_PAGE.value:
-                page = next(stream)
-                num_page_changes += 1
-                continue
-            elif b == Opcode.RLE.value:
-                offset = next(stream)
-                rle = next(stream)
-                num_rle_bytes += rle
-                for i in range(rle):
-                    self._write(page << 8 | ((offset + i) & 0xff), content)
-                continue
-            elif b == Opcode.TICK.value:
-                continue
-            elif b == Opcode.END_FRAME.value:
-                break
 
-            num_content_stores += 1
-            self._write(page << 8 | b, content)
+class Bitmap:
+    XMAX = None  # type: int
+    YMAX = None  # type: int
 
-        return (
-            num_content_stores, num_content_changes, num_page_changes,
-            num_rle_bytes
-        )
+    def __init__(self, bitmap: np.array = None):
+        if bitmap is None:
+            self.bitmap = np.zeros((self.YMAX, self.XMAX), dtype=bool)
+        else:
+            self.bitmap = bitmap
+
+    def randomize(self) -> None:
+        self.bitmap = np.random.randint(
+            2, size=(self.YMAX, self.XMAX), dtype=bool)
+
+    def pack(self):
+        return Bytemap(self.bitmap)
+
+    @classmethod
+    def from_bytemap(cls, bytemap: Bytemap):
+        return cls(bytemap.unpack())
+
+
+class HGR140Bitmap(Bitmap):
+    XMAX = 140  # double-wide pixels to not worry about colour effects
+    YMAX = 192
+
+    def pack(self):
+        # Double each pixel horizontally
+        return Bytemap(np.repeat(self.bitmap, 2, axis=1))
+
+    @classmethod
+    def from_bytemap(cls, bytemap: Bytemap):
+        # Undouble pixels
+        bm = bytemap.unpack()
+        return cls(
+            np.array(
+                np.delete(bm, np.arange(0, bm.shape[1], 2), axis=1),
+                dtype=np.bool
+            ))
+
+
+class HGRBitmap(Bitmap):
+    XMAX = 280
+    YMAX = 192
+
+
+class DHGRBitmap(Bitmap):
+    XMAX = 560
+    YMAX = 192

server.py

@@ -0,0 +1,26 @@
+import socketserver
+
+ADDR = "192.168.1.15"
+PORT = 20000
+
+
+class ChunkHandler(socketserver.BaseRequestHandler):
+    def handle(self):
+        i = 0
+        while True:
+            i += 1
+            print("sending %d" % i)
+            self.request.sendall(bytes([i % (128 - 32) + 32] * 256))
+
+
+def main():
+    with socketserver.TCPServer(
+            (ADDR, PORT), ChunkHandler, bind_and_activate=False) as server:
+        server.allow_reuse_address = True
+        server.server_bind()
+        server.server_activate()
+        server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

video.py

@@ -0,0 +1,180 @@
+import functools
+from collections import defaultdict
+from typing import Iterator, Set, Tuple
+
+import numpy as np
+
+import opcodes
+import scheduler
+import memory_map
+import screen
+
+
+@functools.lru_cache(None)
+def hamming_weight(n: int) -> int:
+    """Compute hamming weight of 8-bit int"""
+    n = (n & 0x55) + ((n & 0xAA) >> 1)
+    n = (n & 0x33) + ((n & 0xCC) >> 2)
+    n = (n & 0x0F) + ((n & 0xF0) >> 4)
+    return n
+
+
+class Video:
+    """Apple II screen memory map encoding a bitmapped frame."""
+
+    def __init__(self, screen_page: int = 0,
+                 opcode_scheduler: scheduler.OpcodeScheduler = None):
+        self.screen_page = screen_page
+
+        # Initialize empty
+        self.screen = screen.HGRBitmap().pack()  # type: screen.Bytemap
+
+        self.memory_map = memory_map.MemoryMap(screen_page, self.screen)
+
+        self.cycle_counter = opcodes.CycleCounter()
+        self.state = opcodes.State(self.cycle_counter, self.memory_map)
+
+        self.scheduler = (
+                opcode_scheduler or scheduler.HeuristicPageFirstScheduler())
+
+    def update(self, frame: screen.Bitmap,
+               cycle_budget: int, fullness: float) -> Iterator[int]:
+        """Update to match content of frame within provided budget.
+
+        Emits encoded byte stream for rendering the image.
+
+        The byte stream consists of offsets against a selected page (e.g. $20xx)
+        at which to write a selected content byte.  Those selections are
+        controlled by special opcodes emitted to the stream
+
+        Opcodes:
+          SET_CONTENT - new byte to write to screen contents
+          SET_PAGE - set new page to offset against (e.g. $20xx)
+          TICK - tick the speaker
+          DONE - terminate the video decoding
+
+        In order to "make room" for these opcodes we make use of the fact that
+        each page has 2 sets of 8-byte "screen holes", at page offsets
+        0x78-0x7f and 0xf8-0xff.  Currently we only use the latter range as
+        this allows for efficient matching in the critical path of the decoder.
+
+        We group by offsets from page boundary (cf some other more
+        optimal starting point) because STA (..),y has 1 extra cycle if
+        crossing the page boundary.  Though maybe this would be worthwhile if
+        it optimizes the bytestream.
+        """
+
+        self.cycle_counter.reset()
+        # Target screen memory map for new frame
+        target = frame.pack()
+
+        # Estimate number of opcodes that will end up fitting in the cycle
+        # budget.
+        byte_cycles = opcodes.Offset(0).cycles
+        est_opcodes = int(cycle_budget / fullness / byte_cycles)
+
+        # Sort by highest xor weight and take the estimated number of change
+        # operations
+        # TODO: changes should be a class
+        changes = list(
+            sorted(self.index_changes(self.screen, target), reverse=True)
+        )[:est_opcodes]
+
+        for op in self.scheduler.schedule(changes):
+            yield from self.state.emit(op)
+
+    def index_changes(self, source: screen.Bytemap,
+                      target: screen.Bytemap) -> Set[
+        Tuple[int, int, int, int, int]]:
+        """Transform encoded screen to sequence of change tuples.
+
+        Change tuple is (xor_weight, page, offset, content)
+        """
+
+        changes = set()
+        # TODO: don't use 256 bytes if XMAX is smaller, or we may compute RLE
+        # over the full page!
+        memmap = defaultdict(lambda: [(0, 0, 0)] * 256)
+
+        it = np.nditer(target.bytemap, flags=['multi_index'])
+        while not it.finished:
+            y, x_byte = it.multi_index
+
+            page, offset = self.memory_map.to_page_offset(x_byte, y)
+
+            src_content = source.bytemap[y][x_byte]
+            target_content = np.asscalar(it[0])
+
+            bits_different = hamming_weight(src_content ^ target_content)
+
+            memmap[page][offset] = (bits_different, src_content, target_content)
+            it.iternext()
+
+        byte_cycles = opcodes.Offset(0).cycles
+
+        for page, offsets in memmap.items():
+            cur_content = None
+            run_length = 0
+            maybe_run = []
+            for offset, data in enumerate(offsets):
+                bits_different, src_content, target_content = data
+
+                # TODO: allowing odd bit errors introduces colour error
+                if maybe_run and hamming_weight(
+                        cur_content ^ target_content) > 2:
+                    # End of run
+
+                    # Decide if it's worth emitting as a run vs single stores
+
+                    # Number of changes in run for which >0 bits differ
+                    num_changes = len([c for c in maybe_run if c[0]])
+                    run_cost = opcodes.RLE(0, run_length).cycles
+                    single_cost = byte_cycles * num_changes
+                    # print("Run of %d cheaper than %d singles" % (
+                    #    run_length, num_changes))
+
+                    # TODO: don't allow too much error to accumulate
+
+                    if run_cost < single_cost:
+                        # Compute median bit value over run
+                        median_bits = np.median(
+                            np.vstack(
+                                np.unpackbits(
+                                    np.array(r[3], dtype=np.uint8)
+                                )
+                                for r in maybe_run
+                            ), axis=0
+                        ) > 0.5
+
+                        typical_content = np.asscalar(np.packbits(median_bits))
+
+                        total_xor = sum(ch[0] for ch in maybe_run)
+                        start_offset = maybe_run[0][2]
+
+                        change = (total_xor, page, start_offset,
+                                  typical_content, run_length)
+                        # print("Found run of %d * %2x at %2x:%2x" % (
+                        #    run_length, cur_content, page, offset - run_length)
+                        #      )
+                        # print(maybe_run)
+                        # print("change =", change)
+                        changes.add(change)
+                    else:
+                        changes.update(ch for ch in maybe_run if ch[0])
+                    maybe_run = []
+                    run_length = 0
+                    cur_content = target_content
+
+                if cur_content is None:
+                    cur_content = target_content
+
+                run_length += 1
+                maybe_run.append(
+                    (bits_different, page, offset, target_content, 1))
+
+        return changes
+
+    def done(self) -> Iterator[int]:
+        """Terminate opcode stream."""
+
+        yield from self.state.emit(opcodes.Terminate())