Finish implementing HGRBitmap support.
- For HGRBitmap introduce a packed representation in the form hhHaaaaaaaABbbbbbbbFff where capitals indicate the location of the palette bit. i.e. for header and footer we include the neighbouring 2 data bits as in DHGR but also the palette bit from that byte, which is necessary to understand how these data bits unpack into dots. The nonstandard ordering of the palette bit for the odd data byte (B) is so that the masking by byte offset produces a contiguous sequence of bits, i.e. the 14-bit masked representation is still dense. - Introduce a to_dots() classmethod that converts from the masked bit representation of dots influenced by a screen byte to the actual sequence of screen dots. For DHGR this is the identity map since there are no shenanigans with palette bits causing dots to shift around. - Add a bunch more unit tests, and add back the Sather tests for HGR artifact colours from palette bit interference, which now all pass! - Reduce the size of the precomputed edit distance matrix by half by exploiting the fact that it is symmetrical under i << N + j <-> j << N + i where N is the size of the masked bit representation (i.e. transposing the original (i, j) -> dist metric matrix).
This commit is contained in:
kris
parent
16c4faa66d
commit
ab29b01d0f
|
|
@ -125,20 +125,6 @@ class MemoryMap:
|
|||
self.page_offset[page - self._page_start][offset] = val
|
||||
|
||||
|
||||
@functools.lru_cache(None)
|
||||
def _edit_distances(name: str, palette_id: pal.Palette) -> List[np.ndarray]:
|
||||
"""Load edit distance matrices for masked, shifted byte values.
|
||||
|
||||
This is defined at module level to be a singleton.
|
||||
"""
|
||||
data = "transcoder/data/%s_palette_%d_edit_distance.pickle.bz2" % (
|
||||
name,
|
||||
palette_id.value
|
||||
)
|
||||
with bz2.open(data, "rb") as ed:
|
||||
return pickle.load(ed) # type: List[np.ndarray]
|
||||
|
||||
|
||||
class Bitmap:
|
||||
"""Packed 28-bit bitmap representation of (D)HGR screen memory.
|
||||
|
||||
|
|
@ -166,6 +152,9 @@ class Bitmap:
|
|||
# memory byte
|
||||
MASKED_BITS = None # type: np.uint64
|
||||
|
||||
# XXX
|
||||
PHASES = None # type: List[int]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
palette: pal.Palette,
|
||||
|
|
@ -241,6 +230,10 @@ class Bitmap:
|
|||
def _byte_offsets(is_aux: bool) -> Tuple[int, int]:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def to_dots(cls, masked_val: int, byte_offset: int) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
def apply(
|
||||
self,
|
||||
page: int,
|
||||
|
|
@ -253,7 +246,7 @@ class Bitmap:
|
|||
packed_offset = offset // 2
|
||||
|
||||
self.packed[page, packed_offset] = self.masked_update(
|
||||
byte_offset, self.packed[page, packed_offset], np.uint64(value))
|
||||
byte_offset, self.packed[page, packed_offset], value)
|
||||
self._fix_scalar_neighbours(page, packed_offset, byte_offset)
|
||||
|
||||
def _fix_scalar_neighbours(
|
||||
|
|
@ -314,19 +307,43 @@ class Bitmap:
|
|||
shifted_right = np.roll(ary, 1, axis=1)
|
||||
self._fix_column_right(ary, shifted_right)
|
||||
|
||||
@classmethod
|
||||
@functools.lru_cache(None)
|
||||
def edit_distances(self, palette_id: pal.Palette) -> List[np.ndarray]:
|
||||
def edit_distances(cls, palette_id: pal.Palette) -> List[np.ndarray]:
|
||||
"""Load edit distance matrices for masked, shifted byte values."""
|
||||
return _edit_distances(self.NAME, palette_id)
|
||||
|
||||
data = "transcoder/data/%s_palette_%d_edit_distance.pickle.bz2" % (
|
||||
cls.NAME,
|
||||
palette_id.value
|
||||
)
|
||||
with bz2.open(data, "rb") as ed:
|
||||
dist = pickle.load(ed) # type: List[np.ndarray]
|
||||
|
||||
# dist is an upper-triangular matrix of edit_distance(a, b)
|
||||
# encoded as dist[(a << N) + b] = edit_distance(a, b)
|
||||
# Because the distance metric is reflexive,
|
||||
# edit_distance(b, a) = edit_distance(a, b)
|
||||
|
||||
identity = np.arange(2 ** (2 * cls.MASKED_BITS), dtype=np.uint64)
|
||||
# Swap values of form a << N + b to b << N + a
|
||||
transpose = (identity >> cls.MASKED_BITS) + (
|
||||
(identity & np.uint64(2 ** cls.MASKED_BITS - 1)) <<
|
||||
cls.MASKED_BITS)
|
||||
|
||||
for i in range(len(dist)):
|
||||
dist[i][transpose] += dist[i][identity]
|
||||
|
||||
return dist
|
||||
|
||||
@classmethod
|
||||
def mask_and_shift_data(
|
||||
self,
|
||||
cls,
|
||||
data: IntOrArray,
|
||||
byte_offset: int) -> IntOrArray:
|
||||
"""Masks and shifts data into the MASKED_BITS range."""
|
||||
res = (data & self.BYTE_MASKS[byte_offset]) >> (
|
||||
self.BYTE_SHIFTS[byte_offset])
|
||||
assert np.all(res <= 2 ** self.MASKED_BITS)
|
||||
res = (data & cls.BYTE_MASKS[byte_offset]) >> (
|
||||
cls.BYTE_SHIFTS[byte_offset])
|
||||
assert np.all(res <= 2 ** cls.MASKED_BITS)
|
||||
return res
|
||||
|
||||
# TODO: unit tests
|
||||
|
|
@ -420,116 +437,146 @@ class HGRBitmap(Bitmap):
|
|||
MASKED_BITS = np.uint64(14) # 3 + 8 + 3
|
||||
|
||||
HEADER_BITS = np.uint64(3)
|
||||
BODY_BITS = np.uint64(16) # 8 + 8
|
||||
# 7-bits doubled, plus possible shift from palette bit
|
||||
BODY_BITS = np.uint64(15)
|
||||
FOOTER_BITS = np.uint64(3)
|
||||
|
||||
PHASES = [1, 3]
|
||||
|
||||
def __init__(self, palette: pal.Palette, main_memory: MemoryMap):
|
||||
super(HGRBitmap, self).__init__(palette, main_memory, None)
|
||||
|
||||
def _make_header(self, prev_col: IntOrArray) -> IntOrArray:
|
||||
raise NotImplementedError
|
||||
@staticmethod
|
||||
def _make_header(col: IntOrArray) -> IntOrArray:
|
||||
# Header format is bits 5,6,0 of previous byte
|
||||
# i.e. offsets 16, 17, 11
|
||||
|
||||
# return (col & np.uint64(0b111 << 16)) >> np.uint64(16)
|
||||
|
||||
return (
|
||||
(col & np.uint64(0b1 << 11)) >> np.uint64(9) ^ (
|
||||
(col & np.uint64(0b11 << 17)) >> np.uint64(17))
|
||||
)
|
||||
|
||||
def _body(self) -> np.ndarray:
|
||||
raise NotImplementedError
|
||||
# Body is in order
|
||||
# a0 a1 a2 a3 a4 a5 a6 a7 b7 b0 b1 b2 b3 b4 b5 b6
|
||||
# so that a) the header and footer have the same order
|
||||
# across the two byte offsets, and b) so that they
|
||||
# can be extracted as contiguous bit ranges
|
||||
|
||||
def _make_footer(self, next_col: IntOrArray) -> IntOrArray:
|
||||
raise NotImplementedError
|
||||
even = self.main_memory.page_offset[:, 0::2].astype(np.uint64)
|
||||
odd = self.main_memory.page_offset[:, 1::2].astype(np.uint64)
|
||||
|
||||
# XXX move to make_data_tables
|
||||
def _pack(self) -> None:
|
||||
"""Pack main memory into (28+3)-bit uint64 array"""
|
||||
|
||||
# 00000000001111111111222222222233
|
||||
# 01234567890123456789012345678901
|
||||
# AAAABBBBCCCCDDd
|
||||
# AAAABBBBCCCCDd
|
||||
# DDEEEEFFFFGGGGg
|
||||
# dDDEEEEFFFFGGGg
|
||||
|
||||
# Even, P0: store unshifted (0..14)
|
||||
# Even, P1: store shifted << 1 (1..15) (only need 1..14)
|
||||
|
||||
# Odd, P0: store shifted << 14 (14 .. 28) - set bit 14 as bit 0 of next
|
||||
# byte
|
||||
# Odd, p1: store shifted << 15 (15 .. 29) (only need 15 .. 28) - set
|
||||
# bit 13 as bit 0 of next byte
|
||||
|
||||
# Odd overflow only matters for even, P1
|
||||
# - bit 0 is either bit 14 if odd, P0 or bit 13 if odd, P1
|
||||
# - but these both come from the undoubled bit 6.
|
||||
|
||||
main = self.main_memory.page_offset.astype(np.uint64)
|
||||
|
||||
# Double 7-bit pixel data from a into 14-bit fat pixels, and extend MSB
|
||||
# into 15-bits tohandle case when subsequent byte has palette bit set,
|
||||
# i.e. is right-shifted by 1 dot. This only matters for even bytes
|
||||
# with P=0 that are followed by odd bytes with P=1; in other cases
|
||||
# this extra bit will be overwritten.
|
||||
double = (
|
||||
# Bit pos 6
|
||||
((main & 0x40) << 8) + ((main & 0x40) << 7) + (
|
||||
(main & 0x40) << 6)) + (
|
||||
# Bit pos 5
|
||||
((main & 0x20) << 6) + ((main & 0x20) << 5)) + (
|
||||
# Bit pos 4
|
||||
((main & 0x10) << 5) + ((main & 0x10) << 4)) + (
|
||||
# Bit pos 3
|
||||
((main & 0x08) << 4) + ((main & 0x08) << 3)) + (
|
||||
# Bit pos 2
|
||||
((main & 0x04) << 3) + ((main & 0x04) << 2)) + (
|
||||
# Bit pos 1
|
||||
((main & 0x02) << 2) + ((main & 0x02) << 1)) + (
|
||||
# Bit pos 0
|
||||
((main & 0x01) << 1) + (main & 0x01))
|
||||
|
||||
a_even = main[:, ::2]
|
||||
a_odd = main[:, 1::2]
|
||||
|
||||
double_even = double[:, ::2]
|
||||
double_odd = double[:, 1::2]
|
||||
|
||||
# Place even offsets at bits 1..15 (P=1) or 0..14 (P=0)
|
||||
packed = np.where(a_even & 0x80, double_even << 1, double_even)
|
||||
|
||||
# Place off offsets at bits 15..27 (P=1) or 14..27 (P=0)
|
||||
packed = np.where(
|
||||
a_odd & 0x80,
|
||||
np.bitwise_xor(
|
||||
np.bitwise_and(packed, (2 ** 15 - 1)),
|
||||
double_odd << 15
|
||||
),
|
||||
np.bitwise_xor(
|
||||
np.bitwise_and(packed, (2 ** 14 - 1)),
|
||||
double_odd << 14
|
||||
)
|
||||
return (
|
||||
(even << 3) + ((odd & 0x7f) << 12) + ((odd & 0x80) << 4)
|
||||
)
|
||||
|
||||
# Patch up even offsets with P=1 with extended bit from previous odd
|
||||
# column
|
||||
@staticmethod
|
||||
def _make_footer(col: IntOrArray) -> IntOrArray:
|
||||
# Footer format is bits 7,0,1 of next byte
|
||||
# i.e. offsets 10,3,4
|
||||
|
||||
previous_odd = np.roll(a_odd, 1, axis=1).astype(np.uint64)
|
||||
return (
|
||||
(col & np.uint64(0b1 << 10)) >> np.uint64(10) ^ (
|
||||
(col & np.uint64(0b11 << 3)) >> np.uint64(2))
|
||||
) << np.uint64(19)
|
||||
|
||||
packed = np.where(
|
||||
a_even & 0x80,
|
||||
# Truncate to 28-bits and set bit 0 from bit 6 of previous byte
|
||||
np.bitwise_xor(
|
||||
np.bitwise_and(packed, (2 ** 28 - 2)),
|
||||
(previous_odd & (1 << 6)) >> 6
|
||||
),
|
||||
# Truncate to 28-bits
|
||||
np.bitwise_and(packed, (2 ** 28 - 1))
|
||||
)
|
||||
|
||||
# Append first 3 bits of next even byte so we can correctly
|
||||
# decode the effective colours at the end of the 28-bit tuple
|
||||
trailing = np.roll(packed, -1, axis=1).astype(np.uint64)
|
||||
|
||||
packed = np.bitwise_xor(
|
||||
packed,
|
||||
(trailing & 0b111) << 28
|
||||
)
|
||||
|
||||
self.packed = packed
|
||||
# # XXX move to make_data_tables
|
||||
# def _pack(self) -> None:
|
||||
# """Pack main memory into (28+3)-bit uint64 array"""
|
||||
#
|
||||
# # 00000000001111111111222222222233
|
||||
# # 01234567890123456789012345678901
|
||||
# # AAAABBBBCCCCDDd
|
||||
# # AAAABBBBCCCCDd
|
||||
# # DDEEEEFFFFGGGGg
|
||||
# # dDDEEEEFFFFGGGg
|
||||
#
|
||||
# # Even, P0: store unshifted (0..14)
|
||||
# # Even, P1: store shifted << 1 (1..15) (only need 1..14)
|
||||
#
|
||||
# # Odd, P0: store shifted << 14 (14 .. 28) - set bit 14 as bit 0 of next
|
||||
# # byte
|
||||
# # Odd, p1: store shifted << 15 (15 .. 29) (only need 15 .. 28) - set
|
||||
# # bit 13 as bit 0 of next byte
|
||||
#
|
||||
# # Odd overflow only matters for even, P1
|
||||
# # - bit 0 is either bit 14 if odd, P0 or bit 13 if odd, P1
|
||||
# # - but these both come from the undoubled bit 6.
|
||||
#
|
||||
# main = self.main_memory.page_offset.astype(np.uint64)
|
||||
#
|
||||
# # Double 7-bit pixel data from a into 14-bit fat pixels, and extend MSB
|
||||
# # into 15-bits tohandle case when subsequent byte has palette bit set,
|
||||
# # i.e. is right-shifted by 1 dot. This only matters for even bytes
|
||||
# # with P=0 that are followed by odd bytes with P=1; in other cases
|
||||
# # this extra bit will be overwritten.
|
||||
# double = (
|
||||
# # Bit pos 6
|
||||
# ((main & 0x40) << 8) + ((main & 0x40) << 7) + (
|
||||
# (main & 0x40) << 6)) + (
|
||||
# # Bit pos 5
|
||||
# ((main & 0x20) << 6) + ((main & 0x20) << 5)) + (
|
||||
# # Bit pos 4
|
||||
# ((main & 0x10) << 5) + ((main & 0x10) << 4)) + (
|
||||
# # Bit pos 3
|
||||
# ((main & 0x08) << 4) + ((main & 0x08) << 3)) + (
|
||||
# # Bit pos 2
|
||||
# ((main & 0x04) << 3) + ((main & 0x04) << 2)) + (
|
||||
# # Bit pos 1
|
||||
# ((main & 0x02) << 2) + ((main & 0x02) << 1)) + (
|
||||
# # Bit pos 0
|
||||
# ((main & 0x01) << 1) + (main & 0x01))
|
||||
#
|
||||
# a_even = main[:, ::2]
|
||||
# a_odd = main[:, 1::2]
|
||||
#
|
||||
# double_even = double[:, ::2]
|
||||
# double_odd = double[:, 1::2]
|
||||
#
|
||||
# # Place even offsets at bits 1..15 (P=1) or 0..14 (P=0)
|
||||
# packed = np.where(a_even & 0x80, double_even << 1, double_even)
|
||||
#
|
||||
# # Place off offsets at bits 15..27 (P=1) or 14..27 (P=0)
|
||||
# packed = np.where(
|
||||
# a_odd & 0x80,
|
||||
# np.bitwise_xor(
|
||||
# np.bitwise_and(packed, (2 ** 15 - 1)),
|
||||
# double_odd << 15
|
||||
# ),
|
||||
# np.bitwise_xor(
|
||||
# np.bitwise_and(packed, (2 ** 14 - 1)),
|
||||
# double_odd << 14
|
||||
# )
|
||||
# )
|
||||
#
|
||||
# # Patch up even offsets with P=1 with extended bit from previous odd
|
||||
# # column
|
||||
#
|
||||
# previous_odd = np.roll(a_odd, 1, axis=1).astype(np.uint64)
|
||||
#
|
||||
# packed = np.where(
|
||||
# a_even & 0x80,
|
||||
# # Truncate to 28-bits and set bit 0 from bit 6 of previous byte
|
||||
# np.bitwise_xor(
|
||||
# np.bitwise_and(packed, (2 ** 28 - 2)),
|
||||
# (previous_odd & (1 << 6)) >> 6
|
||||
# ),
|
||||
# # Truncate to 28-bits
|
||||
# np.bitwise_and(packed, (2 ** 28 - 1))
|
||||
# )
|
||||
#
|
||||
# # Append first 3 bits of next even byte so we can correctly
|
||||
# # decode the effective colours at the end of the 28-bit tuple
|
||||
# trailing = np.roll(packed, -1, axis=1).astype(np.uint64)
|
||||
#
|
||||
# packed = np.bitwise_xor(
|
||||
# packed,
|
||||
# (trailing & 0b111) << 28
|
||||
# )
|
||||
#
|
||||
# self.packed = packed
|
||||
|
||||
@staticmethod
|
||||
@functools.lru_cache(None)
|
||||
|
|
@ -547,6 +594,76 @@ class HGRBitmap(Bitmap):
|
|||
assert not is_aux
|
||||
return 0, 1
|
||||
|
||||
@staticmethod
|
||||
@functools.lru_cache(None)
|
||||
def _double_pixels(int7: int) -> int:
|
||||
|
||||
# Input bit 6 is repeated 3 times in case the neighbouring byte is
|
||||
# delayed (right-shifted by one dot) due to the palette bit being set.
|
||||
# Care needs to be taken to mask this out when overwriting.
|
||||
double = (
|
||||
# Bit pos 6
|
||||
((int7 & 0x40) << 8) + ((int7 & 0x40) << 7) + (
|
||||
(int7 & 0x40) << 6) +
|
||||
# Bit pos 5
|
||||
((int7 & 0x20) << 6) + ((int7 & 0x20) << 5) +
|
||||
# Bit pos 4
|
||||
((int7 & 0x10) << 5) + ((int7 & 0x10) << 4) + (
|
||||
# Bit pos 3
|
||||
((int7 & 0x08) << 4) + ((int7 & 0x08) << 3) +
|
||||
# Bit pos 2
|
||||
((int7 & 0x04) << 3) + ((int7 & 0x04) << 2) +
|
||||
# Bit pos 1
|
||||
((int7 & 0x02) << 2) + ((int7 & 0x02) << 1) +
|
||||
# Bit pos 0
|
||||
((int7 & 0x01) << 1) + (int7 & 0x01))
|
||||
)
|
||||
|
||||
return double
|
||||
|
||||
@classmethod
|
||||
def to_dots(cls, masked_val: int, byte_offset: int) -> int:
|
||||
|
||||
# Assert 14-bit representation
|
||||
assert (masked_val & (2 ** 14 - 1)) == masked_val
|
||||
|
||||
# Unpack hhHaaaaaaaABbbbbbbbFff
|
||||
|
||||
# --> hhhaaaaaaaaaaaaaabbbb (P=0, P=0, P=0)
|
||||
# hhhaaaaaaaaaaaaaabbbb (P=1, P=0, P=0)
|
||||
# hhhhaaaaaaaaaaaaabbbb (P=1, P=1, P=0)
|
||||
# hhhhaaaaaaaaaaaaaabbb (P=1, P=1, P=1)
|
||||
|
||||
# Take top 3 bits from header (plus duplicated MSB) not 4, because if it
|
||||
# is palette-shifted then we don't know what is in bit 0
|
||||
h = (masked_val & 0b111) << 5
|
||||
hp = (h & 0x80) >> 7
|
||||
res = cls._double_pixels(h & 0x7f) >> (11 - hp)
|
||||
|
||||
if byte_offset == 0:
|
||||
# Offset 0: hhHaaaaaaaABbb
|
||||
b = (masked_val >> 3) & 0xff
|
||||
bp = (b & 0x80) >> 7
|
||||
else:
|
||||
# Offset 1: aaABbbbbbbbFff
|
||||
bp = (masked_val >> 3) & 0x01
|
||||
b = ((masked_val >> 4) & 0x7f) ^ (bp << 7)
|
||||
|
||||
# Mask out current contents in case we are overwriting the extended
|
||||
# high bit from previous screen byte
|
||||
res &= ~((2 ** 14 - 1) << (3 + bp))
|
||||
res ^= cls._double_pixels(b & 0x7f) << (3 + bp)
|
||||
|
||||
f = ((masked_val >> 12) & 0b11) ^ (
|
||||
(masked_val >> 11) & 0b01) << 7
|
||||
fp = (f & 0x80) >> 7
|
||||
|
||||
# Mask out current contents in case we are overwriting the extended
|
||||
# high bit from previous screen byte
|
||||
res &= ~((2 ** 4 - 1) << (17 + fp))
|
||||
res ^= cls._double_pixels(f & 0x7f) << (17 + fp)
|
||||
return res & (2 ** 21 - 1)
|
||||
|
||||
# XXX test
|
||||
@staticmethod
|
||||
def masked_update(
|
||||
|
|
@ -558,12 +675,22 @@ class HGRBitmap(Bitmap):
|
|||
Does not patch up headers/footers of neighbouring columns.
|
||||
"""
|
||||
|
||||
# Mask out 8-bit value where update will go
|
||||
masked_value = old_value & (
|
||||
~np.uint64(0xff << (8 * byte_offset + 3)))
|
||||
if byte_offset == 0:
|
||||
# Mask out 8-bit value where update will go
|
||||
masked_value = old_value & (~np.uint64(0xff << 3))
|
||||
|
||||
update = new_value << np.uint64(8 * byte_offset + 3)
|
||||
return masked_value ^ update
|
||||
update = np.uint64(new_value) << np.uint64(3)
|
||||
return masked_value ^ update
|
||||
else:
|
||||
# Mask out 8-bit value where update will go
|
||||
masked_value = old_value & (~np.uint64(0xff << 11))
|
||||
|
||||
# shift palette bit into position 0
|
||||
shifted_new_value = (
|
||||
(new_value & 0x7f) << 1) ^ (
|
||||
(new_value & 0x80) >> 7)
|
||||
update = np.uint64(shifted_new_value) << np.uint64(11)
|
||||
return masked_value ^ update
|
||||
|
||||
|
||||
class DHGRBitmap(Bitmap):
|
||||
|
|
@ -596,6 +723,16 @@ class DHGRBitmap(Bitmap):
|
|||
|
||||
MASKED_BITS = np.uint64(13)
|
||||
|
||||
# NTSC clock phase at first masked bit
|
||||
# Each DHGR byte offset has the same range of int13 possible
|
||||
# values and nominal colour pixels, but with different initial
|
||||
# phases:
|
||||
# AUX 0: 0 (1 at start of 3-bit header)
|
||||
# MAIN 0: 3 (0)
|
||||
# AUX 1: 2 (3)
|
||||
# MAIN 1: 1 (2)
|
||||
PHASES = [1, 0, 3, 2]
|
||||
|
||||
def _body(self) -> np.ndarray:
|
||||
# Palette bit is unused for DHGR so mask it out
|
||||
aux = (self.aux_memory.page_offset & 0x7f).astype(np.uint64)
|
||||
|
|
@ -651,6 +788,13 @@ class DHGRBitmap(Bitmap):
|
|||
|
||||
return offsets
|
||||
|
||||
@classmethod
|
||||
def to_dots(cls, masked_val: int, byte_offset: int) -> int:
|
||||
# For DHGR the 13-bit masked value is already a 13-bit dot sequence
|
||||
# so no need to transform it.
|
||||
|
||||
return masked_val
|
||||
|
||||
@staticmethod
|
||||
def masked_update(
|
||||
byte_offset: int,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue