mirror of
https://github.com/KrisKennaway/ii-vision.git
synced 2024-06-01 14:41:36 +00:00
Integrated audio + video player!
- Introduce a new Movie() class that multiplexes audio and video. - Every N audio frames we grab a new video frame and begin pulling opcodes from the audio and video streams - Grab frames from the input video using bmp2dhr if the .BIN file does not already exist. Run bmp2dhr in a background thread to not block encoding - move the output byte streaming from Video to Movie - For now, manually clip updates to pages > 56 since the client doesn't support them yet The way we encode video is now: - iterate in descending order over update_priority - begin a new (page, content) opcode - for all of the other offset bytes in that page, compute the error between the candidate content byte and the target content byte - iterate over offsets in order of increasing error and decreasing update_priority to fill out the remaining opcode
This commit is contained in:
kris
parent
f133bb0008
commit
c00300147e
59
audio.py
59
audio.py
|
|
@ -9,23 +9,21 @@ import video
|
|||
|
||||
|
||||
class Audio:
|
||||
def encode_audio(self, audio):
|
||||
for a in audio:
|
||||
a = max(-30, min(a * 2, 32)) + 34
|
||||
page = random.randint(32, 56)
|
||||
content = random.randint(0,255)
|
||||
offsets = [random.randint(0, 255) for _ in range(4)]
|
||||
yield opcodes.TICK_OPCODES[(a, page)](content, offsets)
|
||||
def __init__(
|
||||
self, filename: str, normalization: float = 1.0):
|
||||
self.filename = filename
|
||||
self.normalization = normalization
|
||||
|
||||
# TODO: take into account that the available range is slightly offset
|
||||
# as fraction of total cycle count?
|
||||
self._tick_range = [4, 66]
|
||||
self.cycles_per_tick = 73
|
||||
|
||||
def main():
|
||||
filename = "Computer Chronicles - 06x05 - The Apple II.mp4"
|
||||
# TODO: round to divisor of video frame rate
|
||||
self.sample_rate = 14340 # int(1024. * 1024 / self.cycles_per_tick)
|
||||
|
||||
s = video.Video(frame_rate=None)
|
||||
au = Audio()
|
||||
|
||||
with audioread.audio_open(filename) as f:
|
||||
with open("out.bin", "wb") as out:
|
||||
def audio_stream(self):
|
||||
with audioread.audio_open(self.filename) as f:
|
||||
for buf in f.read_data(128 * 1024):
|
||||
print(f.channels, f.samplerate, f.duration)
|
||||
|
||||
|
|
@ -33,24 +31,29 @@ def main():
|
|||
'float32').reshape((f.channels, -1), order='F')
|
||||
|
||||
a = librosa.core.to_mono(data)
|
||||
a = librosa.resample(a, f.samplerate, 14000).flatten()
|
||||
a = librosa.resample(a, f.samplerate,
|
||||
self.sample_rate).flatten()
|
||||
|
||||
# Normalize to 95%ile
|
||||
# norm = max(
|
||||
# abs(np.percentile(a, 5, axis=0)),
|
||||
# abs(np.percentile(a, 95, axis=0))
|
||||
# )
|
||||
# print(min(a),max(a))
|
||||
# print(norm)
|
||||
a /= 16384 # normalize to -1.0 .. 1.0
|
||||
a *= self.normalization
|
||||
|
||||
# XXX how to estimate normalization without reading whole file?
|
||||
norm = 12000
|
||||
# Convert to -16 .. 16
|
||||
a = (a * 16).astype(np.int)
|
||||
a = np.clip(a, -15, 16)
|
||||
|
||||
a /= norm # librosa.util.normalize(a)
|
||||
a = (a * 32).astype(np.int)
|
||||
yield from a
|
||||
|
||||
out.write(bytes(s.emit_stream(au.encode_audio(a))))
|
||||
out.write(bytes(s.done()))
|
||||
|
||||
def main():
|
||||
filename = "Computer Chronicles - 06x05 - The Apple II.mp4"
|
||||
|
||||
s = video.Video(frame_rate=None)
|
||||
au = Audio(filename, normalization=3)
|
||||
|
||||
with open("out.bin", "wb") as out:
|
||||
for b in s.emit_stream(au.encode_audio()):
|
||||
out.write(bytearray([b]))
|
||||
out.write(bytes(s.done()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
import os
|
||||
import threading
|
||||
import queue
|
||||
import subprocess
|
||||
from typing import Iterable
|
||||
|
||||
from PIL import Image
|
||||
|
|
@ -7,22 +11,64 @@ import numpy as np
|
|||
import screen
|
||||
|
||||
|
||||
def frame_grabber(filename: str) -> Iterable[np.array]:
|
||||
"""Yields a sequence of Image frames in original resolution."""
|
||||
for frame_array in skvideo.io.vreader(filename):
|
||||
yield Image.fromarray(frame_array)
|
||||
|
||||
|
||||
def hgr140_frame_grabber(filename: str) -> Iterable[screen.MemoryMap]:
|
||||
bm_cls = screen.HGR140Bitmap
|
||||
for frame in skvideo.io.vreader(filename):
|
||||
im = Image.fromarray(frame)
|
||||
im = im.resize((bm_cls.XMAX, bm_cls.YMAX))
|
||||
im = im.convert("1")
|
||||
im = np.array(im)
|
||||
for frame in frame_grabber(filename):
|
||||
frame = frame.resize((bm_cls.XMAX, bm_cls.YMAX))
|
||||
frame = frame.convert("1")
|
||||
frame = np.array(frame)
|
||||
|
||||
yield bm_cls(im).to_bytemap().to_memory_map(screen_page=1)
|
||||
yield bm_cls(frame).to_bytemap().to_memory_map(screen_page=1)
|
||||
|
||||
|
||||
def bmp_frame_grabber(filename: str) -> Iterable[screen.MemoryMap]:
|
||||
idx = 0
|
||||
def bmp2dhr_frame_grabber(filename: str) -> Iterable[screen.MemoryMap]:
|
||||
"""Encode frame to HGR using bmp2dhr"""
|
||||
|
||||
frame_dir = filename.split(".")[0]
|
||||
try:
|
||||
os.mkdir(frame_dir)
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
q = queue.Queue(maxsize=10)
|
||||
|
||||
def worker():
|
||||
for idx, frame in enumerate(frame_grabber(filename)):
|
||||
outfile = "%s/%08dC.BIN" % (frame_dir, idx)
|
||||
bmpfile = "%s/%08d.bmp" % (frame_dir, idx)
|
||||
|
||||
try:
|
||||
os.stat(outfile)
|
||||
except FileNotFoundError:
|
||||
frame = frame.resize((280, 192))
|
||||
frame.save(bmpfile)
|
||||
|
||||
subprocess.call(
|
||||
["/usr/local/bin/bmp2dhr", bmpfile, "hgr", "D9"])
|
||||
|
||||
os.remove(bmpfile)
|
||||
|
||||
frame = np.fromfile(outfile, dtype=np.uint8)
|
||||
q.put(frame)
|
||||
|
||||
q.put(None)
|
||||
|
||||
t = threading.Thread(target=worker)
|
||||
t.start()
|
||||
|
||||
while True:
|
||||
fn = "%s-%08dC.BIN" % (filename, idx)
|
||||
frame = np.fromfile(fn, dtype=np.uint8)
|
||||
frame = q.get()
|
||||
|
||||
if frame is None:
|
||||
break
|
||||
|
||||
yield screen.FlatMemoryMap(screen_page=1, data=frame).to_memory_map()
|
||||
idx += 1
|
||||
q.task_done()
|
||||
|
||||
t.join()
|
||||
44
main.py
44
main.py
|
|
@ -1,53 +1,27 @@
|
|||
import frame_grabber
|
||||
import movie
|
||||
import opcodes
|
||||
import screen
|
||||
import video
|
||||
|
||||
MAX_OUT = 100 * 1024 * 1024
|
||||
MAX_OUT = 10 * 1024 * 1024
|
||||
VIDEO_FPS = 30
|
||||
APPLE_FPS = 7
|
||||
APPLE_FPS = 30
|
||||
|
||||
|
||||
def main():
|
||||
#frames = frame_grabber.hgr140_frame_grabber(
|
||||
# "Computer Chronicles - 06x05 - The Apple II.mp4")
|
||||
filename = "Computer Chronicles - 06x05 - The Apple II.mp4"
|
||||
|
||||
frames = frame_grabber.bmp_frame_grabber("cc/CC")
|
||||
|
||||
bytes_out = 0
|
||||
sims = []
|
||||
out_frames = 0
|
||||
|
||||
s = video.Video(APPLE_FPS)
|
||||
|
||||
# Assert that the opcode stream reconstructs the same screen
|
||||
ds = video.Video()
|
||||
decoder = opcodes.Decoder(s.state)
|
||||
m = movie.Movie(filename, audio_normalization=3.0)
|
||||
|
||||
with open("out.bin", "wb") as out:
|
||||
for idx, frame in enumerate(frames):
|
||||
if idx % (VIDEO_FPS // APPLE_FPS):
|
||||
continue
|
||||
for bytes_out, b in enumerate(m.emit_stream(m.encode())):
|
||||
out.write(bytearray([b]))
|
||||
|
||||
stream = bytes(s.emit_stream(s.encode_frame(frame)))
|
||||
|
||||
bytes_out += len(stream)
|
||||
bytes_left = MAX_OUT - bytes_out
|
||||
|
||||
sim = screen.bitmap_similarity(
|
||||
screen.HGRBitmap.from_bytemap(s.memory_map.to_bytemap()).bitmap,
|
||||
screen.HGRBitmap.from_bytemap(frame.to_bytemap()).bitmap)
|
||||
sims.append(sim)
|
||||
out_frames += 1
|
||||
print("Frame %d, %d bytes, similarity = %f" % (
|
||||
idx, len(stream), sim))
|
||||
out.write(stream[:bytes_left])
|
||||
|
||||
if bytes_left <= 0:
|
||||
out.write(bytes(s.done()))
|
||||
if bytes_out >= MAX_OUT:
|
||||
break
|
||||
|
||||
print("Median similarity: %f" % sorted(sims)[out_frames//2])
|
||||
out.write(bytes(m.done()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
236
video.py
236
video.py
|
|
@ -1,4 +1,6 @@
|
|||
import functools
|
||||
import heapq
|
||||
import random
|
||||
from typing import Iterator, Tuple, Iterable
|
||||
|
||||
import numpy as np
|
||||
|
|
@ -72,9 +74,13 @@ class Video:
|
|||
|
||||
CLOCK_SPEED = 1024 * 1024
|
||||
|
||||
def __init__(self, frame_rate: int = 15, screen_page: int = 1,
|
||||
opcode_scheduler: scheduler.OpcodeScheduler = None):
|
||||
def __init__(
|
||||
self,
|
||||
frame_rate: int = 30,
|
||||
screen_page: int = 1,
|
||||
opcode_scheduler: scheduler.OpcodeScheduler = None):
|
||||
self.screen_page = screen_page
|
||||
self.frame_rate = frame_rate
|
||||
|
||||
# Initialize empty
|
||||
self.memory_map = screen.MemoryMap(
|
||||
|
|
@ -83,118 +89,17 @@ class Video:
|
|||
self.scheduler = (
|
||||
opcode_scheduler or scheduler.HeuristicPageFirstScheduler())
|
||||
|
||||
self.cycle_counter = opcodes.CycleCounter()
|
||||
|
||||
# Accumulates pending edit weights across frames
|
||||
self.update_priority = np.zeros((32, 256), dtype=np.int)
|
||||
|
||||
self.state = opcodes.State(
|
||||
self.cycle_counter, self.memory_map, self.update_priority)
|
||||
|
||||
self.frame_rate = frame_rate
|
||||
self.stream_pos = 0
|
||||
if self.frame_rate:
|
||||
self.cycles_per_frame = self.CLOCK_SPEED // self.frame_rate
|
||||
else:
|
||||
self.cycles_per_frame = None
|
||||
|
||||
self._last_op = opcodes.Nop()
|
||||
|
||||
def encode_frame(self, target: screen.MemoryMap) -> Iterator[
|
||||
opcodes.Opcode]:
|
||||
"""Update to match content of frame within provided budget.
|
||||
|
||||
Emits encoded byte stream for rendering the image.
|
||||
|
||||
XXX update
|
||||
|
||||
The byte stream consists of offsets against a selected page (e.g. $20xx)
|
||||
at which to write a selected content byte. Those selections are
|
||||
controlled by special opcodes emitted to the stream
|
||||
|
||||
Opcodes:
|
||||
SET_CONTENT - new byte to write to screen contents
|
||||
SET_PAGE - set new page to offset against (e.g. $20xx)
|
||||
TICK - tick the speaker
|
||||
DONE - terminate the video decoding
|
||||
|
||||
We group by offsets from page boundary (cf some other more
|
||||
optimal starting point) because STA (..),y has 1 extra cycle if
|
||||
crossing the page boundary. Though maybe this would be worthwhile if
|
||||
it optimizes the bytestream.
|
||||
"""
|
||||
|
||||
# TODO: changes should be a class
|
||||
changes = self._index_changes(self.memory_map, target)
|
||||
|
||||
yield from self.scheduler.schedule(changes)
|
||||
|
||||
@functools.lru_cache()
|
||||
def _rle_cycles(self, run_length):
|
||||
return opcodes.RLE(0, run_length).cycles
|
||||
|
||||
def _index_page(self, bits_different, target_content):
|
||||
byte_cycles = opcodes.Store(0).cycles
|
||||
|
||||
cur_content = None
|
||||
run_length = 0
|
||||
run = []
|
||||
|
||||
# Number of changes in run for which >0 bits differ
|
||||
num_changes_in_run = 0
|
||||
|
||||
# Total weight of differences accumulated in run
|
||||
total_update_priority_in_run = 0
|
||||
|
||||
def end_run():
|
||||
# Decide if it's worth emitting as a run vs single stores
|
||||
run_cost = self._rle_cycles(run_length)
|
||||
single_cost = byte_cycles * num_changes_in_run
|
||||
# print("Run of %d cheaper than %d singles" % (
|
||||
# run_length, num_changes_in_run))
|
||||
|
||||
if run_cost < single_cost:
|
||||
start_offset = run[0][1]
|
||||
|
||||
# print("Found run of %d * %2x at %2x" % (
|
||||
# run_length, cur_content, offset - run_length)
|
||||
# )
|
||||
# print(run)
|
||||
yield (
|
||||
total_update_priority_in_run, start_offset, cur_content,
|
||||
run_length
|
||||
)
|
||||
else:
|
||||
for ch in run:
|
||||
if ch[0]:
|
||||
yield ch
|
||||
|
||||
for offset in range(256):
|
||||
bd = bits_different[offset]
|
||||
tc = target_content[offset]
|
||||
if run and cur_content != tc:
|
||||
# End of run
|
||||
|
||||
yield from end_run()
|
||||
|
||||
run = []
|
||||
run_length = 0
|
||||
num_changes_in_run = 0
|
||||
total_update_priority_in_run = 0
|
||||
cur_content = tc
|
||||
|
||||
if cur_content is None:
|
||||
cur_content = tc
|
||||
|
||||
run_length += 1
|
||||
run.append((bd, offset, tc, 1))
|
||||
if bd:
|
||||
num_changes_in_run += 1
|
||||
total_update_priority_in_run += bd
|
||||
|
||||
if run:
|
||||
# End of run
|
||||
yield from end_run()
|
||||
yield from self._index_changes(self.memory_map, target)
|
||||
|
||||
def _index_changes(
|
||||
self,
|
||||
|
|
@ -209,9 +114,13 @@ class Video:
|
|||
diff_weights = np.zeros((32, 256), dtype=np.uint8)
|
||||
|
||||
it = np.nditer(
|
||||
source.page_offset ^ target.page_offset,
|
||||
flags=['multi_index'])
|
||||
source.page_offset ^ target.page_offset, flags=['multi_index'])
|
||||
while not it.finished:
|
||||
# If no diff, don't need to bother
|
||||
if not it[0]:
|
||||
it.iternext()
|
||||
continue
|
||||
|
||||
diff_weights[it.multi_index] = edit_weight(
|
||||
source.page_offset[it.multi_index],
|
||||
target.page_offset[it.multi_index],
|
||||
|
|
@ -225,51 +134,82 @@ class Video:
|
|||
|
||||
self.update_priority += diff_weights
|
||||
|
||||
for page in range(32):
|
||||
for change in self._index_page(
|
||||
self.update_priority[page], target.page_offset[page]):
|
||||
(
|
||||
total_priority_in_run, start_offset, target_content,
|
||||
run_length
|
||||
) = change
|
||||
# Iterate in descending order of update priority and emit tuples
|
||||
# encoding (page, content, [offsets])
|
||||
|
||||
# TODO: handle screen page
|
||||
yield (
|
||||
total_priority_in_run, page + 32, start_offset,
|
||||
target_content, run_length
|
||||
priorities = []
|
||||
it = np.nditer(self.update_priority, flags=['multi_index'])
|
||||
while not it.finished:
|
||||
priority = it[0]
|
||||
if not priority:
|
||||
it.iternext()
|
||||
continue
|
||||
|
||||
page, offset = it.multi_index
|
||||
# Don't use deterministic order for page, offset
|
||||
nonce = random.randint(0,255)
|
||||
heapq.heappush(priorities, (-priority, nonce, page, offset))
|
||||
it.iternext()
|
||||
|
||||
while True:
|
||||
priority, _, page, offset = heapq.heappop(priorities)
|
||||
priority = -priority
|
||||
if page > (56-32):
|
||||
continue
|
||||
offsets = [offset]
|
||||
content = target.page_offset[page, offset]
|
||||
#print("Priority %d: page %d offset %d content %d" % (
|
||||
# priority, page, offset, content))
|
||||
|
||||
# Clear priority for the offset we're emitting
|
||||
self.update_priority[page, offset] = 0
|
||||
|
||||
# Need to find 3 more offsets to fill this opcode
|
||||
|
||||
# Minimize the update_priority delta that would result from
|
||||
# emitting this offset
|
||||
|
||||
# Find offsets that would have largest reduction in diff weight
|
||||
# with this content byte, then order by update priority
|
||||
deltas = {}
|
||||
for o, p in enumerate(self.update_priority[page]):
|
||||
if p == 0:
|
||||
continue
|
||||
|
||||
# If we store content at this offset, what is the new
|
||||
# edit_weight from this content byte to the target
|
||||
delta = edit_weight(
|
||||
content,
|
||||
target.page_offset[page, o],
|
||||
o % 2 == 1
|
||||
)
|
||||
#print("Offset %d prio %d: %d -> %d = %d" % (
|
||||
# o, p, content,
|
||||
# target.page_offset[page, o],
|
||||
# delta
|
||||
#))
|
||||
deltas.setdefault(delta, list()).append((p, o))
|
||||
|
||||
def _emit_bytes(self, _op):
|
||||
# print("%04X:" % self.stream_pos)
|
||||
for b in self.state.emit(self._last_op, _op):
|
||||
yield b
|
||||
self.stream_pos += 1
|
||||
self._last_op = _op
|
||||
for d in sorted(deltas.keys()):
|
||||
#print(d)
|
||||
po = sorted(deltas[d], reverse=True)
|
||||
#print(po)
|
||||
for p, o in po:
|
||||
offsets.append(o)
|
||||
# Clear priority for the offset we're emitting
|
||||
self.update_priority[page, offset] = 0
|
||||
if len(offsets) == 4:
|
||||
break
|
||||
if len(offsets) == 4:
|
||||
break
|
||||
|
||||
# Pad to 4 if we didn't find anything
|
||||
for _ in range(len(offsets), 4):
|
||||
offsets.append(offsets[0])
|
||||
|
||||
#print("Page %d, content %d: offsets %s" % (page+32, content,
|
||||
# offsets))
|
||||
yield (page+32, content, offsets)
|
||||
|
||||
def emit_stream(self, ops: Iterable[opcodes.Opcode]) -> Iterator[int]:
|
||||
self.cycle_counter.reset()
|
||||
for op in ops:
|
||||
# Keep track of where we are in TCP client socket buffer
|
||||
socket_pos = self.stream_pos % 2048
|
||||
if socket_pos >= 2045:
|
||||
# May be about to emit a 3-byte opcode, pad out to last byte
|
||||
# in frame
|
||||
nops = 2047 - socket_pos
|
||||
# print("At position %04x, padding with %d nops" % (
|
||||
# socket_pos, nops))
|
||||
for _ in range(nops):
|
||||
yield from self._emit_bytes(opcodes.Nop())
|
||||
yield from self._emit_bytes(opcodes.Ack())
|
||||
# Ack falls through to nop
|
||||
self._last_op = opcodes.Nop()
|
||||
yield from self._emit_bytes(op)
|
||||
|
||||
if self.cycles_per_frame and (
|
||||
self.cycle_counter.cycles > self.cycles_per_frame):
|
||||
print("Out of cycle budget")
|
||||
return
|
||||
# TODO: pad to cycles_per_frame with NOPs
|
||||
|
||||
def done(self) -> Iterator[int]:
|
||||
"""Terminate opcode stream."""
|
||||
yield from self._emit_bytes(opcodes.Terminate())
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user