Add a TODO to upload more youtube videos showing quality improvements

consistently finding enough additional offsets (~2.5x avg instead of
  >2.9)

- Remove instrumentation for fill rate now that it's served its purpose

README.md

@@ -17,6 +17,8 @@ Dedicated to the memory of [Bob Bishop](https://www.kansasfest.org/2014/11/remem
 
 Sample videos (recording of playback on Apple //gs with RGB monitor, or HDMI via VidHD)
 
+TODO: These are from older versions, for which quality was not as good.
+
 Double Hi-Res:
 - [Try getting this song out of your head](https://youtu.be/S7aNcyojoZI)
 - [Babylon 5 title credits](https://youtu.be/PadKk8n1xY8)
@@ -28,8 +30,6 @@ Older Hi-Res videos:
 - [Paranoimia ft Max Headroom](https://youtu.be/wfdbEyP6v4o)
 - [How many of us still feel about our Apple II's](https://youtu.be/-e5LRcnQF-A)
 
-(These are from older versions, for which quality was not as good)
-
 There may be more on this [YouTube playlist](https://www.youtube.com/playlist?list=PLoAt3SC_duBiIjqK8FBoDG_31nUPB8KBM)
 
 ## Details
@@ -40,7 +40,7 @@ This ends up streaming data at about 100KB/sec of which 56KB/sec are updates to
 
 The video frames are actually encoded at the original frame rate (or optionally by skipping frames), prioritizing differences in the screen content, so the effective frame rate is higher than this if only a fraction of the screen is changing between frames (which is the typical case). 
 
-I'm using the excellent (though under-documented ;) [BMP2DHR](http://www.appleoldies.ca/bmp2dhr/) to encode the input video stream into a sequence of memory maps, then post-processing the frame deltas to prioritize the screen bytes to stream in order to approximate these deltas as closely as possible within the timing budget. 
+I'm using the excellent (though under-documented ;) [BMP2DHR](https://github.com/digarok/b2d) to encode the input video stream into a sequence of memory maps, then post-processing the frame deltas to prioritize the screen bytes to stream in order to approximate these deltas as closely as possible within the timing budget. 
 
 ### KansasFest 2019 presentation
 
@@ -50,32 +50,34 @@ TODO: link video once it is available.
 
 ## Installation
 
-This currently requires python3.7 because some dependencies (e.g. weighted-levenshtein) don't compile with 3.9+, and 3.8
+This currently requires python3.8 because some dependencies (e.g. weighted-levenshtein) don't compile with 3.9+.
-has a [bug](https://bugs.python.org/issue44439) in object pickling.  
 
 ```
-python3.7 -m venv venv
+python3.8 -m venv venv
 source venv/bin/activate
 pip install -r requirements.txt
 ```
 
-To generate the data files required by the transcoder:
+Before you can run the transcoder you need to generate the data files it requires:
 
 ```
 % python transcoder/make_data_tables.py
 ```
 
-This takes about 3 hours on my machine.
+This is a one-time setup.  It takes about 90 minutes on my machine.
 
-TODO: download instructions
+## Sample videos
+
+Some sample videos are available [here](https://www.dropbox.com/sh/kq2ej63smrzruwk/AADZSaqbNuTwAfnPWT6r9TJra?dl=0) for
+streaming (see `server/server.py`)
 
 ## Release Notes
 
 ### v0.3 (17 Jan 2023)
 
 - Fixed an image quality bug in the transcoder
-- Quality of life improvements to installation process
+- Documentation/quality of life improvements to installation process
-- Stop using LFS to store the generated data files in git, they're using up my quota
+- Stop using LFS to store the generated data files in git, they're using up all my quota
 
 ### v0.2 (19 July 2019)
 

requirements.txt

@@ -1,12 +1,30 @@
+appdirs==1.4.4
 audioread==3.0.0
+certifi==2022.12.7
+cffi==1.15.1
+charset-normalizer==3.0.1
 colormath==3.0.0
+decorator==5.1.1
 etaprogress==1.1.1
+idna==3.4
+importlib-metadata==6.0.0
+joblib==1.2.0
 librosa==0.9.2
-networkx==2.6.3
+llvmlite==0.39.1
-numpy==1.21.6
+networkx==3.0
+numba==0.56.4
+numpy==1.22.4  # Until colormath supports 1.23+
+packaging==23.0
 Pillow==9.4.0
-scikit-learn==1.0.2
+pooch==1.6.0
+pycparser==2.21
+requests==2.28.2
+resampy==0.4.2
+scikit-learn==1.2.0
 scikit-video==1.1.11
-scipy==1.7.3
+scipy==1.10.0
 soundfile==0.11.0
+threadpoolctl==3.1.0
+urllib3==1.26.14
 weighted-levenshtein==0.2.1
+zipp==3.11.0

transcoder/audio.py

@@ -64,8 +64,8 @@ class Audio:
     def _normalization(self, read_bytes=1024 * 1024 * 10):
         """Read first read_bytes of audio stream and compute normalization.
 
-        We compute the 2.5th and 97.5th percentiles i.e. only 2.5% of samples
+        We normalize based on the 0.5th and 99.5th percentiles, i.e. only <1% of
-        will clip.
+        samples will clip.
 
         :param read_bytes:
         :return:
@@ -77,7 +77,7 @@ class Audio:
                 if len(raw) > read_bytes:
                     break
         a = self._decode(f, raw)
-        norm = np.max(np.abs(np.percentile(a, [2.5, 97.5])))
+        norm = np.max(np.abs(np.percentile(a, [0.5, 99.5])))
 
         return 16384. / norm
 

transcoder/make_data_tables.py

@@ -113,7 +113,7 @@ def compute_edit_distance(
         edp: EditDistanceParams,
         bitmap_cls: Type[screen.Bitmap],
         nominal_colours: Type[colours.NominalColours]
-):
+) -> np.ndarray:
     """Computes edit distance matrix between all pairs of pixel strings.
 
     Enumerates all possible values of the masked bit representation from
@@ -131,44 +131,45 @@ def compute_edit_distance(
 
     bitrange = np.uint64(2 ** bits)
 
-    edit = []
+    edit = np.zeros(
-    for _ in range(len(bitmap_cls.BYTE_MASKS)):
+        shape=(len(bitmap_cls.BYTE_MASKS), np.uint64(bitrange * bitrange)),
-        edit.append(
+        dtype=np.uint16)
-            np.zeros(shape=np.uint64(bitrange * bitrange), dtype=np.uint16))
 
-    # Matrix is symmetrical with zero diagonal so only need to compute upper
+    bar = ProgressBar(
-    # triangle
+        bitrange * (bitrange - 1) / 2 * len(bitmap_cls.PHASES), max_width=80)
-    bar = ProgressBar((bitrange * (bitrange - 1)) / 2, max_width=80)
 
     num_dots = bitmap_cls.MASKED_DOTS
 
     cnt = 0
     for i in range(np.uint64(bitrange)):
-        for j in range(i):
+        pair_base = np.uint64(i) << bits
-            cnt += 1
+        for o, ph in enumerate(bitmap_cls.PHASES):
+            # Compute this in the outer loop since it's invariant under j
+            first_dots = bitmap_cls.to_dots(i, byte_offset=o)
+            first_pixels = pixel_string(
+                colours.dots_to_nominal_colour_pixel_values(
+                    num_dots, first_dots, nominal_colours,
+                    init_phase=ph)
+            )
 
-            if cnt % 10000 == 0:
+            # Matrix is symmetrical with zero diagonal so only need to compute
-                bar.numerator = cnt
+            # upper triangle
-                print(bar, end='\r')
+            for j in range(i):
-                sys.stdout.flush()
+                cnt += 1
+                if cnt % 100000 == 0:
+                    bar.numerator = cnt
+                    print(bar, end='\r')
+                    sys.stdout.flush()
 
-            pair = (np.uint64(i) << bits) + np.uint64(j)
+                pair = pair_base + np.uint64(j)
 
-            for o, ph in enumerate(bitmap_cls.PHASES):
-                first_dots = bitmap_cls.to_dots(i, byte_offset=o)
                 second_dots = bitmap_cls.to_dots(j, byte_offset=o)
-
-                first_pixels = pixel_string(
-                    colours.dots_to_nominal_colour_pixel_values(
-                        num_dots, first_dots, nominal_colours,
-                        init_phase=ph)
-                )
                 second_pixels = pixel_string(
                     colours.dots_to_nominal_colour_pixel_values(
                         num_dots, second_dots, nominal_colours,
                         init_phase=ph)
                 )
-                edit[o][pair] = edit_distance(
+                edit[o, pair] = edit_distance(
                     edp, first_pixels, second_pixels, error=False)
 
     return edit
@@ -183,10 +184,9 @@ def make_edit_distance(
     """Write file containing (D)HGR edit distance matrix for a palette."""
 
     dist = compute_edit_distance(edp, bitmap_cls, nominal_colours)
-    data = "transcoder/data/%s_palette_%d_edit_distance.pickle.bz2" % (
+    data = "transcoder/data/%s_palette_%d_edit_distance.npz" % (
         bitmap_cls.NAME, pal.ID.value)
-    with bz2.open(data, "wb", compresslevel=9) as out:
+    np.savez_compressed(data, edit_distance=dist)
-        pickle.dump(dist, out, protocol=pickle.HIGHEST_PROTOCOL)
 
 
 def main():

transcoder/screen.py

@@ -342,15 +342,13 @@ class Bitmap:
 
     @classmethod
     @functools.lru_cache(None)
-    def edit_distances(cls, palette_id: pal.Palette) -> List[np.ndarray]:
+    def edit_distances(cls, palette_id: pal.Palette) -> np.ndarray:
         """Load edit distance matrices for masked, shifted byte values."""
 
-        data = "transcoder/data/%s_palette_%d_edit_distance.pickle.bz2" % (
+        data = "transcoder/data/%s_palette_%d_edit_distance.npz" % (
-            cls.NAME,
+            cls.NAME, palette_id.value
-            palette_id.value
         )
-        with bz2.open(data, "rb") as ed:
+        dist = np.load(data)['edit_distance']
-            dist = pickle.load(ed)  # type: List[np.ndarray]
 
         # dist is an upper-triangular matrix of edit_distance(a, b)
         # encoded as dist[(a << N) + b] = edit_distance(a, b)
@@ -363,8 +361,8 @@ class Bitmap:
                 (identity & np.uint64(2 ** cls.MASKED_BITS - 1)) <<
                 cls.MASKED_BITS)
 
-        for i in range(len(dist)):
+        for i in range(dist.shape[0]):
-            dist[i][transpose] += dist[i][identity]
+            dist[i, transpose] += dist[i, identity]
 
         return dist
 
@@ -741,6 +739,7 @@ class HGRBitmap(Bitmap):
         return double
 
     @classmethod
+    @functools.lru_cache(None)
     def to_dots(cls, masked_val: int, byte_offset: int) -> int:
         """Convert masked representation to bit sequence of display dots.