Fixing lots of decompression bugs.

Platform/Apple/tools/PackPartitions/src/org/demo/PackPartitions.groovy

@@ -509,11 +509,12 @@ class PackPartitions
     // Transform the LZ4 format to something we call "LZ4M", where the small offsets are stored
     // as one byte instead of two. In our data, that's about 1/3 of the offsets.
     //
-    def recompress(data, inLen, expOutLen)
+    def recompress(data, inLen, uncompData, uncompLen)
     {
         def outLen = 0
         def sp = 0
         def dp = 0
+        def cksum = 0
         while (true) 
         {
             assert dp <= sp
@@ -534,12 +535,14 @@ class PackPartitions
             }
             
             if (debugCompression)
-                println String.format("Literal: len=\$%x.", literalLen)
+                println String.format("Literal: ptr=\$%x, len=\$%x.", (0x4400+sp), literalLen)
             
             // Copy the literal bytes
             outLen += literalLen
-            for ( ; literalLen > 0; --literalLen)
+            for ( ; literalLen > 0; --literalLen) {
+                cksum ^= data[sp]
                 data[dp++] = data[sp++]
+            }
             
             // The last block has only literals, and no match
             if (sp == inLen)
@@ -558,6 +561,13 @@ class PackPartitions
                 data[dp++] = (offset >> 7) & 0xFF
             }
             
+            // If checksums are enabled, output the checksum so far
+            if (offset < 128 && ADD_COMP_CHECKSUMS) {
+                if (debugCompression)
+                    println String.format("    [chksum=\$%x]", cksum & 0xFF)
+                data[dp++] = (byte) cksum
+            }
+            
             // The match length might get extended
             if (matchLen == 15) {
                 while (true) {
@@ -573,18 +583,21 @@ class PackPartitions
             if (debugCompression)
                 println String.format("Match: offset=\$%x, len=\$%x.", offset, matchLen)
             
-            // We do nothing with the match bytes except count them
+            // We do nothing with the match bytes except add them to the checksum
-            outLen += matchLen
+            (0..<matchLen).each {
+                cksum ^= uncompData[outLen]
+                ++outLen
+            }
         }
         
-        // If checksums are enabled, add an exclusive-or checksum to the end.
+        // If checksums are enabled, output the final checksum
         if (ADD_COMP_CHECKSUMS) {
-            def cksum = 0
+            if (debugCompression)
-            (0..<dp).each { cksum ^= data[it] }
+                println String.format("Final cksum: \$%x", cksum & 0xFF)
             data[dp++] = (byte) cksum
         }
-        
+
-        assert outLen == expOutLen
+        assert outLen == uncompLen
         return dp
     }
     
@@ -606,7 +619,7 @@ class PackPartitions
                                                 compressedData, 0, maxCompressedLen)
                                             
         // Then recompress to LZ4M (pretty much always smaller)
-        def recompressedLen = recompress(compressedData, compressedLen, uncompressedLen)
+        def recompressedLen = recompress(compressedData, compressedLen, uncompressedData, uncompressedLen)
         
         // If we saved at least 20 bytes, take the compressed version.
         if ((uncompressedLen - recompressedLen) >= 20) {

Platform/Apple/virtual/src/core/mem.s

@@ -17,6 +17,7 @@
 MAX_SEGS	= 96
 
 DO_COMP_CHECKSUMS = 1		; during compression debugging
+DEBUG_DECOMP = 0
 
 ; Zero page temporary variables
 tmp		= $2	; len 2
@@ -1190,7 +1191,7 @@ lz4Decompress: !zone
 +
 }
 
-	!if DEBUG { jsr .debug1 }
+	!if DEBUG_DECOMP { jsr .debug1 }
 	jsr readToBuf		; read first pages into buffer
 	ldx #<clrAuxWr		; start by assuming write to main mem
 	ldy #<clrAuxRd		; and read from main mem
@@ -1238,7 +1239,7 @@ lz4Decompress: !zone
 	jsr .longLen		; special marker: extend the length
 +	sta ucLen		; record resulting length (lo byte)
 .goLit:
-	!if DEBUG { jsr .debug2 }
+	!if DEBUG_DECOMP { jsr .debug2 }
 .auxWr1	sta setAuxWr		; this gets self-modified depending on if target is in main or aux mem
 .litCopy:			; loop to copy the literals
 	+LOAD_YSRC		; grab a literal source byte
@@ -1264,11 +1265,7 @@ lz4Decompress: !zone
 	lda #0              	; have we finished all pages?
 	bne .decodeMatch	; no, keep going
 	pla			; toss unused match length
-	!if DO_COMP_CHECKSUMS {
+	!if DO_COMP_CHECKSUMS { jsr .verifyCksum }
-	lda checksum		; get computed checksum
-	beq +			; should be zero, because compressor stores checksum byte as part of stream
-	brk			; checksum doesn't match -- abort!
-+	}
 	rts			; all done!
 	; Now that we've finished with the literals, decode the match section
 .decodeMatch:
@@ -1276,12 +1273,13 @@ lz4Decompress: !zone
 	sta tmp			; save for later
 	cmp #0
 	bmi .far		; if hi bit is set, there will be a second byte
+	!if DO_COMP_CHECKSUMS { jsr .verifyCksum }
 	lda #0			; otherwise, second byte is assumed to be zero
 	beq .doInv		; always taken
 .far:	+LOAD_YSRC		; grab second byte of offset
 	asl tmp			; toss the unused hi bit of the lo byte
  	lsr			; shift out lo bit of the hi byte
-	rol tmp			; to fill in the hi bit of the lo byte
+	ror tmp			; to fill in the hi bit of the lo byte
 .doInv:	sta tmp+1		; got the hi byte of the offset now
 	lda #0			; calculate zero minus the offset, to obtain ptr diff
 	sec
@@ -1290,7 +1288,7 @@ lz4Decompress: !zone
 	lda .dstStore2+2	; same with hi byte of offset
 	sbc tmp+1
 	sta .srcLoad+2		; to hi byte of offsetted pointer
-	!if DEBUG { jsr .debug4 }
+	!if DEBUG_DECOMP { jsr .debug3 }
 .getMatchLen:
 	pla			; recover the token byte
 	and #$F			; mask to get just the match length
@@ -1300,7 +1298,7 @@ lz4Decompress: !zone
 	bne +			; if not, no need to extend length
 	jsr .longLen		; need to extend the length
 +	sty tmp			; save index to source pointer, so we can use Y...
-	!if DEBUG { sta ucLen : jsr .debug3 }
+	!if DEBUG_DECOMP { sta ucLen : jsr .debug4 }
 	tay			; ...to count bytes
 .auxWr2	sta setAuxWr		; self-modified earlier, based on isAuxCmd
 	jsr .matchCopy		; copy match bytes (aux->aux, or main->main)
@@ -1337,8 +1335,8 @@ lz4Decompress: !zone
 .matchShadow_end = *
 	; Subroutine called when length token = $F, to extend the length by additional bytes
 .longLen:
-	sta ucLen		; save what we got so far
+-	sta ucLen		; save what we got so far
--	+LOAD_YSRC		; get another byte
+	+LOAD_YSRC		; get another byte
 	cmp #$FF		; check for special there-is-more marker byte
 	php			; save result of that
 	clc
@@ -1349,6 +1347,24 @@ lz4Decompress: !zone
 	beq -			; if it was $FF, go back for more len bytes
 	rts
 
+	!if DO_COMP_CHECKSUMS {
+.verifyCksum:
+	+LOAD_YSRC
+	!if DEBUG_DECOMP {
+	+prStr : !text "cksum exp=",0
+	pha
+	jsr prbyte
+	+prStr : !text " got=",0
+	+prByte checksum
+	+crout
+	pla
+	}
+	cmp checksum		; get computed checksum
+	beq +			; should be zero, because compressor stores checksum byte as part of stream
+	brk			; checksum doesn't match -- abort!
++	rts
+	}
+
 nextSrcPage:
 	pha			; save byte that was loaded
 	inc pSrc+1		; go to next page
@@ -1356,7 +1372,11 @@ nextSrcPage:
 	cmp #>diskBufEnd	; did we reach end of buffer?
 	bne +			; if not, we're done
 	sta clrAuxWr		; buffer is in main mem
+	txa
+	pha
 	jsr readToBuf		; read more pages
+	pla
+	tax
 .auxWr3	sta setAuxWr		; go back to writing aux mem (self-modified for aux or main)
 +	pla			; restore loaded byte
 	rts
@@ -1381,7 +1401,7 @@ setupDecomp:
 	bpl -			; loop until we grab them all (including byte 0)
 	rts
 	
-!if DEBUG {
+!if DEBUG_DECOMP {
 .debug1	+prStr : !text "Decompressing: isComp=",0
 	+prByte isCompressed
 	+prStr : !text "isAux=",0
@@ -1405,12 +1425,7 @@ setupDecomp:
 	+prWord ucLen
 	+crout
 	rts
-.debug3	+prStr : !text "len=",0
+.debug3	+prStr : !text "Match src=",0
-	+prWord ucLen
-	+crout
-	+waitKey
-	rts
-.debug4	+prStr : !text "Match src=",0
 	txa			; calculate src address with X (not Y!) as offset
 	clc
 	adc .srcLoad+1
@@ -1438,6 +1453,11 @@ setupDecomp:
 	sta pTmp+1
 	+prWord pTmp		; and print it
 	rts
+.debug4	+prStr : !text "len=",0
+	+prWord ucLen
+	+crout
+	+waitKey
+	rts
 }
 
 ;------------------------------------------------------------------------------