Bump version

More optimizations to the 6809 LZSA depackers

README.md

@@ -71,9 +71,11 @@ License:
 
 8-bit assembly code:
 
-* Z80 decompressors (size- and speed-optimized) written by [introspec](https://github.com/specke)
+* Z80 decompressors (size- and speed-optimized) written by [introspec](https://github.com/specke) with optimizations by [uniabis](https://github.com/uniabis)
 * 6502 and 8088 size-optimized improvements by [Peter Ferrie](https://github.com/peterferrie)
+* 6502 speed-optimized decompressor by [John Brandwood](https://github.com/jbrandwood)
 * 8088 speed-optimized decompressor by [Jim Leonard](https://github.com/mobygamer)
+* 6809 decompressors (Tandy CoCo, Thomson MO/TO, Dragon 32/64..) optimized by [Doug Masten](https://github.com/dougmasten)
 
 External links:
 

asm/6809/unlzsa1.s

@@ -0,0 +1,104 @@
+;  unlzsa1.s - 6809 decompression routine for raw LZSA1 - 110 bytes
+;  compress with lzsa -r <original_file> <compressed_file>
+;
+;  in:  x = start of compressed data
+;       y = start of decompression buffer
+;  out: y = end of decompression buffer + 1
+;
+;  Copyright (C) 2020 Emmanuel Marty
+;
+;  This software is provided 'as-is', without any express or implied
+;  warranty.  In no event will the authors be held liable for any damages
+;  arising from the use of this software.
+;
+;  Permission is granted to anyone to use this software for any purpose,
+;  including commercial applications, and to alter it and redistribute it
+;  freely, subject to the following restrictions:
+;
+;  1. The origin of this software must not be misrepresented; you must not
+;     claim that you wrote the original software. If you use this software
+;     in a product, an acknowledgment in the product documentation would be
+;     appreciated but is not required.
+;  2. Altered source versions must be plainly marked as such, and must not be
+;     misrepresented as being the original software.
+;  3. This notice may not be removed or altered from any source distribution.
+
+decompress_lzsa1 equ lz1token
+
+lz1bigof lda ,x+           ; O set: load MSB 16-bit (negative, signed) offest
+lz1gotof leau d,y          ; put backreference start address in U (dst+offset)
+
+         puls b            ; restore token
+
+         clra              ; clear A (high part of match length)
+         andb #$0F         ; isolate MMMM (embedded match length)
+         addb #$03         ; add MIN_MATCH_SIZE
+         cmpb #$12         ; MATCH_RUN_LEN?
+         bne lz1gotln      ; no, we have the full match length, go copy
+
+         addb ,x+          ; add extra match length byte + MIN_MATCH_SIZE + MATCH_RUN_LEN
+         bcc lz1gotln      ; if no overflow, we have the full length
+         bne lz1midln
+
+         ldb ,x+           ; load 16-bit len in D (low part in B, high in A)
+         lda ,x+           ; (little endian)
+         bne lz1gotln      ; check if we hit EOD (16-bit length = 0)
+         tstb
+         bne lz1gotln      ; go copy matched bytes if not
+
+         rts               ; done, bail
+
+lz1midln tfr b,a           ; copy high part of len into A
+         ldb ,x+           ; grab low 8 bits of len in B
+
+lz1gotln pshs x            ; save source compressed data pointer
+         tfr d,x           ; copy match length to X
+
+lz1cpymt lda ,u+           ; copy matched byte
+         sta ,y+
+         leax -1,x         ; decrement X
+         bne lz1cpymt      ; loop until all matched bytes are copied
+
+         puls x            ; restore source compressed data pointer
+
+lz1token ldb ,x+           ; load next token into B: O|LLL|MMMM
+         pshs b            ; save it
+
+         andb #$70         ; isolate LLL (embedded literals count) in B
+         beq lz1nolt       ; skip if no literals
+         cmpb #$70         ; LITERALS_RUN_LEN?
+         bne lz1declt      ; if not, we have the complete count, go unshift
+
+         ldb ,x+           ; load extra literals count byte
+         addb #$07         ; add LITERALS_RUN_LEN
+         bcc lz1gotla      ; if no overflow, we got the complete count, copy
+         bne lz1midlt
+
+         ldb ,x+           ; load low 8 bits of little-endian literals count
+         lda ,x+           ; load high 8 bits of literal count
+         bra lz1gotlt      ; we now have the complete count, go copy
+
+lz1midlt tfr b,a           ; copy high part of literals count into A
+         ldb ,x+           ; load low 8 bits of literals count
+         bra lz1gotlt      ; we now have the complete count, go copy
+
+lz1declt lsrb              ; shift literals count into place
+         lsrb
+         lsrb
+         lsrb
+lz1gotla clra              ; clear A (high part of literals count)
+
+lz1gotlt tfr x,u
+         tfr d,x           ; transfer 16-bit count into X
+lz1cpylt lda ,u+           ; copy literal byte
+         sta ,y+
+         leax -1,x         ; decrement X and update Z flag
+         bne lz1cpylt      ; loop until all literal bytes are copied
+         tfr u,x
+
+lz1nolt  ldb ,x+           ; load either 8-bit or LSB 16-bit offset (negative, signed)
+         lda ,s            ; get token again, don't pop it from the stack
+         bmi lz1bigof      ; test O bit (small or large offset)
+
+         lda #$ff          ; set high 8 bits
+         bra lz1gotof

asm/6809/unlzsa1b.s

@@ -0,0 +1,106 @@
+;  unlzsa1b.s - 6809 backward decompression routine for raw LZSA1 - 113 bytes
+;  compress with lzsa -r -b <original_file> <compressed_file>
+;
+;  in:  x = last byte of compressed data
+;       y = last byte of decompression buffer
+;  out: y = first byte of decompressed data
+;
+;  Copyright (C) 2020 Emmanuel Marty
+;
+;  This software is provided 'as-is', without any express or implied
+;  warranty.  In no event will the authors be held liable for any damages
+;  arising from the use of this software.
+;
+;  Permission is granted to anyone to use this software for any purpose,
+;  including commercial applications, and to alter it and redistribute it
+;  freely, subject to the following restrictions:
+;
+;  1. The origin of this software must not be misrepresented; you must not
+;     claim that you wrote the original software. If you use this software
+;     in a product, an acknowledgment in the product documentation would be
+;     appreciated but is not required.
+;  2. Altered source versions must be plainly marked as such, and must not be
+;     misrepresented as being the original software.
+;  3. This notice may not be removed or altered from any source distribution.
+
+decompress_lzsa1
+         leax 1,x
+         leay 1,y
+         bra lz1token
+
+lz1bigof ldd ,--x          ; O set: load long 16 bit (negative, signed) offset
+lz1gotof nega              ; reverse sign of offset in D
+         negb
+         sbca #0
+         leau d,y          ; put backreference start address in U (dst+offset)
+
+         puls b            ; restore token
+
+         clra              ; clear A (high part of match length)
+         andb #$0F         ; isolate MMMM (embedded match length)
+         addb #$03         ; add MIN_MATCH_SIZE
+         cmpb #$12         ; MATCH_RUN_LEN?
+         bne lz1gotln      ; no, we have the full match length, go copy
+
+         addb ,-x          ; add extra match length byte + MIN_MATCH_SIZE + MATCH_RUN_LEN
+         bcc lz1gotln      ; if no overflow, we have the full length
+         bne lz1midln
+
+         ldd ,--x          ; load 16-bit len in D (low part in B, high in A)
+         bne lz1gotln      ; check if we hit EOD (16-bit length = 0)
+
+         rts               ; done, bail
+
+lz1midln tfr b,a           ; copy high part of len into A
+         ldb ,-x           ; grab low 8 bits of len in B
+
+lz1gotln pshs x            ; save source compressed data pointer
+         tfr d,x           ; copy match length to X
+
+lz1cpymt lda ,-u           ; copy matched byte
+         sta ,-y
+         leax -1,x         ; decrement X
+         bne lz1cpymt      ; loop until all matched bytes are copied
+
+         puls x            ; restore source compressed data pointer
+
+lz1token ldb ,-x           ; load next token into B: O|LLL|MMMM
+         pshs b            ; save it
+
+         andb #$70         ; isolate LLL (embedded literals count) in B
+         beq lz1nolt       ; skip if no literals
+         cmpb #$70         ; LITERALS_RUN_LEN?
+         bne lz1declt      ; if not, we have the complete count, go unshift
+
+         ldb ,-x           ; load extra literals count byte
+         addb #$07         ; add LITERALS_RUN_LEN
+         bcc lz1gotla      ; if no overflow, we got the complete count, copy
+         bne lz1midlt
+
+         ldd ,--x          ; load 16 bit count in D (low part in B, high in A)
+         bra lz1gotlt      ; we now have the complete count, go copy
+
+lz1midlt tfr b,a           ; copy high part of literals count into A
+         ldb ,-x           ; load low 8 bits of literals count
+         bra lz1gotlt      ; we now have the complete count, go copy
+
+lz1declt lsrb              ; shift literals count into place
+         lsrb
+         lsrb
+         lsrb
+
+lz1gotla clra              ; clear A (high part of literals count)
+lz1gotlt tfr x,u
+         tfr d,x           ; transfer 16-bit count into X
+lz1cpylt lda ,-u           ; copy literal byte
+         sta ,-y
+         leax -1,x         ; decrement X and update Z flag
+         bne lz1cpylt      ; loop until all literal bytes are copied
+         tfr u,x
+
+lz1nolt  ldb ,s            ; get token again, don't pop it from the stack
+         bmi lz1bigof      ; test O bit (small or large offset)
+
+         ldb ,-x           ; O clear: load 8 bit (negative, signed) offset
+         lda #$ff          ; set high 8 bits
+         bra lz1gotof

asm/6809/unlzsa2.s

@@ -0,0 +1,149 @@
+;  unlzsa2.s - 6809 decompression routine for raw LZSA2 - 172 bytes
+;  compress with lzsa -f2 -r <original_file> <compressed_file>
+;
+;  in:  x = start of compressed data
+;       y = start of decompression buffer
+;  out: y = end of decompression buffer + 1
+;
+;  Copyright (C) 2020 Emmanuel Marty
+;
+;  This software is provided 'as-is', without any express or implied
+;  warranty.  In no event will the authors be held liable for any damages
+;  arising from the use of this software.
+;
+;  Permission is granted to anyone to use this software for any purpose,
+;  including commercial applications, and to alter it and redistribute it
+;  freely, subject to the following restrictions:
+;
+;  1. The origin of this software must not be misrepresented; you must not
+;     claim that you wrote the original software. If you use this software
+;     in a product, an acknowledgment in the product documentation would be
+;     appreciated but is not required.
+;  2. Altered source versions must be plainly marked as such, and must not be
+;     misrepresented as being the original software.
+;  3. This notice may not be removed or altered from any source distribution.
+
+decompress_lzsa2
+         lsr <lz2nibct,pcr ; reset nibble available flag
+
+lz2token ldb ,x+           ; load next token into B: XYZ|LL|MMM
+         pshs b            ; save it
+
+         andb #$18         ; isolate LL (embedded literals count) in B
+         beq lz2nolt       ; skip if no literals
+         cmpb #$18         ; LITERALS_RUN_LEN_V2?
+         bne lz2declt      ; if not, we have the complete count, go unshift
+
+         bsr lz2nibl       ; get extra literals length nibble in B
+         addb #$03         ; add LITERALS_RUN_LEN_V2
+         cmpb #$12         ; LITERALS_RUN_LEN_V2 + 15 ?
+         bne lz2gotla      ; if not, we have the full literals count, go copy
+
+         addb ,x+          ; add extra literals count byte + LITERALS_RUN_LEN + 15
+         bcc lz2gotla      ; if no overflow, we got the complete count, copy
+
+         ldb ,x+           ; load low 8 bits of little-endian literals count
+         lda ,x+           ; load high 8 bits of literal count
+         bra lz2gotlt      ; we now have the complete count, go copy
+
+lz2declt lsrb              ; shift literals count into place
+         lsrb
+         lsrb
+
+lz2gotla clra              ; clear A (high part of literals count)
+lz2gotlt tfr x,u
+         tfr d,x           ; transfer 16-bit count into X
+lz2cpylt lda ,u+           ; copy literal byte
+         sta ,y+
+         leax -1,x         ; decrement X and update Z flag
+         bne lz2cpylt      ; loop until all literal bytes are copied
+         tfr u,x
+
+lz2nolt  ldb ,s            ; get token again, don't pop it from the stack
+
+         lslb              ; push token's X flag bit into carry
+         bcs lz2replg      ; if token's X bit is set, rep or large offset
+
+         lslb              ; push token's Y flag bit into carry
+         sex               ; push token's Z flag bit into reg A (carry flag is not effected)
+         bcs lz2offs9      ; if token's Y bit is set, 9 bits offset
+
+         bsr lz2nibl       ; get offset nibble in B
+         lsla              ; retrieve token's Z flag bit and push into carry
+
+         rolb              ; shift Z flag from carry into bit 0 of B
+         eorb #$e1         ; set bits 5-7 of offset, reverse bit 0
+         sex               ; set bits 8-15 of offset to $FF
+         bra lz2gotof
+
+lz2offs9 deca              ; set bits 9-15 of offset, reverse bit 8
+         ldb ,x+           ; load low 8 bits of (negative, signed) offset
+         bra lz2gotof
+
+lz2nibct fcb $00           ; nibble ready flag
+
+lz2nibl  ldb #$aa
+         lsr <lz2nibct,pcr  ; nibble ready?
+         bcs lz2gotnb
+
+         inc <lz2nibct,pcr  ; flag nibble as ready for next time
+         ldb ,x+           ; load two nibbles
+         stb <lz2nibl+1,pcr ; store nibble for next time (low 4 bits)
+
+         lsrb              ; shift 4 high bits of nibble down
+         lsrb
+         lsrb
+         lsrb
+
+lz2gotnb andb #$0f         ; only keep low 4 bits
+lz2done  rts
+
+lz2replg lslb              ; push token's Y flag bit into carry
+         bcs lz2rep16      ; if token's Y bit is set, rep or 16 bit offset
+
+         sex               ; push token's Z flag bit into reg A
+         bsr lz2nibl       ; get offset nibble in B
+         lsla              ; push token's Z flag bit into carry
+         rolb              ; shift Z flag from carry into bit 0 of B
+         eorb #$e1         ; set bits 13-15 of offset, reverse bit 8
+         tfr b,a           ; copy bits 8-15 of offset into A
+         suba #$02         ; substract 512 from offset
+         ldb ,x+           ; load low 8 bits of (negative, signed) offset
+         bra lz2gotof
+
+lz2rep16 bmi lz2repof      ; if token's Z flag bit is set, rep match
+         ldd ,x++          ; load high then low 8 bits of offset
+
+lz2gotof std <lz2repof+2,pcr ; store match offset
+lz2repof leau $aaaa,y      ; put backreference start address in U (dst+offset)
+
+         puls b            ; restore token
+
+         clra              ; clear A (high part of match length)
+         andb #$07         ; isolate MMM (embedded match length)
+         addb #$02         ; add MIN_MATCH_SIZE_V2
+         cmpb #$09         ; MIN_MATCH_SIZE_V2 + MATCH_RUN_LEN_V2?
+         bne lz2gotln      ; no, we have the full match length, go copy
+
+         bsr lz2nibl       ; get offset nibble in B
+         addb #$09         ; add MIN_MATCH_SIZE_V2 + MATCH_RUN_LEN_V2
+         cmpb #$18         ; MIN_MATCH_SIZE_V2 + MATCH_RUN_LEN_V2 + 15?
+         bne lz2gotln      ; if not, we have the full match length, go copy
+
+         addb ,x+          ; add extra length byte + MIN_MATCH_SIZE_V2 + MATCH_RUN_LEN_V2 + 15
+         bcc lz2gotln      ; if no overflow, we have the full length
+         beq lz2done       ; detect EOD code
+
+         ldb ,x+           ; load 16-bit len in D (low part in B, high in A)
+         lda ,x+           ; (little endian)
+
+lz2gotln pshs x            ; save source compressed data pointer
+         tfr d,x           ; copy match length to X
+
+lz2cpymt lda ,u+           ; copy matched byte
+         sta ,y+
+         leax -1,x         ; decrement X
+         bne lz2cpymt      ; loop until all matched bytes are copied
+
+         puls x            ; restore source compressed data pointer
+         lbra lz2token     ; go decode next token

asm/6809/unlzsa2b.s

@@ -0,0 +1,154 @@
+;  unlzsa2b.s - 6809 backward decompression routine for raw LZSA2 - 174 bytes
+;  compress with lzsa -f2 -r -b <original_file> <compressed_file>
+;
+;  in:  x = last byte of compressed data
+;       y = last byte of decompression buffer
+;  out: y = first byte of decompressed data
+;
+;  Copyright (C) 2020 Emmanuel Marty
+;
+;  This software is provided 'as-is', without any express or implied
+;  warranty.  In no event will the authors be held liable for any damages
+;  arising from the use of this software.
+;
+;  Permission is granted to anyone to use this software for any purpose,
+;  including commercial applications, and to alter it and redistribute it
+;  freely, subject to the following restrictions:
+;
+;  1. The origin of this software must not be misrepresented; you must not
+;     claim that you wrote the original software. If you use this software
+;     in a product, an acknowledgment in the product documentation would be
+;     appreciated but is not required.
+;  2. Altered source versions must be plainly marked as such, and must not be
+;     misrepresented as being the original software.
+;  3. This notice may not be removed or altered from any source distribution.
+
+decompress_lzsa2
+         lsr <lz2nibct,pcr ; reset nibble available flag
+         leax 1,x
+         leay 1,y
+
+lz2token ldb ,-x           ; load next token into B: XYZ|LL|MMM
+         pshs b            ; save it
+
+         andb #$18         ; isolate LLL (embedded literals count) in B
+         beq lz2nolt       ; skip if no literals
+         cmpb #$18         ; LITERALS_RUN_LEN_V2?
+         bne lz2declt      ; if not, we have the complete count, go unshift
+
+         bsr lz2nibl       ; get extra literals length nibble in B
+         addb #$03         ; add LITERALS_RUN_LEN_V2
+         cmpb #$12         ; LITERALS_RUN_LEN_V2 + 15 ?
+         bne lz2gotla      ; if not, we have the full literals count, go copy
+
+         addb ,-x         ; add extra literals count byte + LITERALS_RUN_LEN + 15
+         bcc lz2gotla      ; if no overflow, we got the complete count, copy
+
+         ldd ,--x          ; load 16 bit count in D (low part in B, high in A)
+         bra lz2gotlt      ; we now have the complete count, go copy
+
+lz2declt lsrb              ; shift literals count into place
+         lsrb
+         lsrb
+lz2gotla clra              ; clear A (high part of literals count)
+
+lz2gotlt tfr x,u
+         tfr d,x           ; transfer 16-bit count into X
+lz2cpylt lda ,-u           ; copy literal byte
+         sta ,-y
+         leax -1,x         ; decrement X and update Z flag
+         bne lz2cpylt      ; loop until all literal bytes are copied
+         tfr u,x
+
+lz2nolt  ldb ,s            ; get token again, don't pop it from the stack
+
+         lslb              ; push token's X flag bit into carry
+         bcs lz2replg      ; if token's X bit is set, rep or large offset
+
+         lslb              ; push token's Y flag bit into carry
+         sex               ; push token's Z flag bit into reg A (carry flag is not effected)
+         bcs lz2offs9      ; if token's Y bit is set, 9 bits offset
+
+         bsr lz2nibl       ; get offset nibble in B
+         lsla              ; retrieve token's Z flag bit and push into carry
+
+         rolb              ; shift Z flag from carry into bit 0 of B
+         eorb #$e1         ; set bits 5-7 of offset, reverse bit 0
+         sex               ; set bits 8-15 of offset to $FF
+         bra lz2gotof
+
+lz2offs9 deca               ; set bits 9-15 of offset, reverse bit 8
+         bra lz2lowof
+
+lz2nibct fcb $00           ; nibble ready flag
+
+lz2nibl  ldb #$aa
+         lsr <lz2nibct,pcr  ; nibble ready?
+         bcs lz2gotnb
+
+         inc <lz2nibct,pcr  ; flag nibble as ready for next time
+         ldb ,-x           ; load two nibbles
+         stb <lz2nibl+1,pcr ; store nibble for next time (low 4 bits)
+
+         lsrb              ; shift 4 high bits of nibble down
+         lsrb
+         lsrb
+         lsrb
+
+lz2gotnb andb #$0f         ; only keep low 4 bits
+lz2done  rts
+
+lz2replg lslb              ; push token's Y flag bit into carry
+         bcs lz2rep16      ; if token's Y bit is set, rep or 16 bit offset
+
+         sex               ; push token's Z flag bit into reg A
+         bsr lz2nibl       ; get offset nibble in B
+         lsla              ; retrieve token's Z flag bit and push into carry
+
+         rolb              ; shift Z flag from carry into bit 0 of B
+         eorb #$e1         ; set bits 13-15 of offset, reverse bit 8
+         tfr b,a           ; copy bits 8-15 of offset into A
+         suba #$02         ; substract 512 from offset
+         bra lz2lowof
+
+lz2rep16 bmi lz2repof      ; if token's Z flag bit is set, rep match
+
+         lda ,-x           ; load high 8 bits of (negative, signed) offset
+lz2lowof ldb ,-x           ; load low 8 bits of offset
+
+lz2gotof nega              ; reverse sign of offset in D
+         negb
+         sbca #0
+         std <lz2repof+2,pcr ; store match offset
+
+lz2repof leau $aaaa,y      ; put backreference start address in U (dst+offset)
+
+         puls b            ; restore token
+
+         clra              ; clear A (high part of match length)
+         andb #$07         ; isolate MMM (embedded match length)
+         addb #$02         ; add MIN_MATCH_SIZE_V2
+         cmpb #$09         ; MIN_MATCH_SIZE_V2 + MATCH_RUN_LEN_V2?
+         bne lz2gotln      ; no, we have the full match length, go copy
+
+         bsr lz2nibl       ; get offset nibble in B
+         addb #$09         ; add MIN_MATCH_SIZE_V2 + MATCH_RUN_LEN_V2
+         cmpb #$18         ; MIN_MATCH_SIZE_V2 + MATCH_RUN_LEN_V2 + 15?
+         bne lz2gotln      ; if not, we have the full match length, go copy
+
+         addb ,-x          ; add extra length byte + MIN_MATCH_SIZE_V2 + MATCH_RUN_LEN_V2 + 15
+         bcc lz2gotln      ; if no overflow, we have the full length
+         beq lz2done       ; detect EOD code
+
+         ldd ,--x          ; load 16-bit len in D (low part in B, high in A)
+
+lz2gotln pshs x            ; save source compressed data pointer
+         tfr d,x           ; copy match length to X
+
+lz2cpymt lda ,-u           ; copy matched byte
+         sta ,-y
+         leax -1,x         ; decrement X
+         bne lz2cpymt      ; loop until all matched bytes are copied
+
+         puls x            ; restore source compressed data pointer
+         lbra lz2token     ; go decode next token

src/lzsa.c

@@ -48,7 +48,7 @@
 #define OPT_RAW_BACKWARD   8
 #define OPT_STATS          16
 
-#define TOOL_VERSION "1.2.2"
+#define TOOL_VERSION "1.3.4"
 
 /*---------------------------------------------------------------------------*/
 

src/shrink_block_v1.c

@@ -157,66 +157,69 @@ static inline int lzsa_get_offset_cost_v1(const unsigned int nMatchOffset) {
  * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
  */
 static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset, const int nReduce) {
-   lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << MATCHES_PER_ARRIVAL_SHIFT);
+   lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << ARRIVALS_PER_POSITION_SHIFT);
    const int nMinMatchSize = pCompressor->min_match_size;
    const int nFavorRatio = (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) ? 1 : 0;
+   const int nModeSwitchPenalty = nFavorRatio ? 0 : MODESWITCH_PENALTY;
    const int nDisableScore = nReduce ? 0 : (2 * BLOCK_SIZE);
    int i, j, n;
 
    if ((nEndOffset - nStartOffset) > BLOCK_SIZE) return;
 
-   memset(arrival + (nStartOffset << MATCHES_PER_ARRIVAL_SHIFT), 0, sizeof(lzsa_arrival) * ((nEndOffset - nStartOffset + 1) << MATCHES_PER_ARRIVAL_SHIFT));
+   memset(arrival + (nStartOffset << ARRIVALS_PER_POSITION_SHIFT), 0, sizeof(lzsa_arrival) * ((nEndOffset - nStartOffset + 1) << ARRIVALS_PER_POSITION_SHIFT));
 
-   arrival[nStartOffset << MATCHES_PER_ARRIVAL_SHIFT].from_slot = -1;
+   arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT].from_slot = -1;
 
    for (i = nStartOffset; i != nEndOffset; i++) {
+      lzsa_arrival* cur_arrival = &arrival[i << ARRIVALS_PER_POSITION_SHIFT];
       int m;
 
-      for (j = 0; j < NMATCHES_PER_ARRIVAL_V1 && arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].from_slot; j++) {
-         int nPrevCost = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].cost;
+      for (j = 0; j < NARRIVALS_PER_POSITION_V1 && cur_arrival[j].from_slot; j++) {
+         int nPrevCost = cur_arrival[j].cost;
          int nCodingChoiceCost = nPrevCost + 8 /* literal */;
-         int nScore = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].score + 1;
-         int nNumLiterals = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].num_literals + 1;
+         int nScore = cur_arrival[j].score + 1;
+         int nNumLiterals = cur_arrival[j].num_literals + 1;
 
          if (nNumLiterals == LITERALS_RUN_LEN_V1 || nNumLiterals == 256 || nNumLiterals == 512) {
             nCodingChoiceCost += 8;
          }
 
-         if (!nFavorRatio && nNumLiterals == 1)
-            nCodingChoiceCost += MODESWITCH_PENALTY;
+         if (nNumLiterals == 1)
+            nCodingChoiceCost += nModeSwitchPenalty;
 
-         for (n = 0; n < NMATCHES_PER_ARRIVAL_V1 /* we only need the literals + short match cost + long match cost cases */; n++) {
-            lzsa_arrival *pDestArrival = &arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n];
+         lzsa_arrival *pDestSlots = &arrival[(i + 1) << ARRIVALS_PER_POSITION_SHIFT];
+         for (n = 0; n < NARRIVALS_PER_POSITION_V1 /* we only need the literals + short match cost + long match cost cases */; n++) {
+            lzsa_arrival *pDestArrival = &pDestSlots[n];
 
             if (pDestArrival->from_slot == 0 ||
                nCodingChoiceCost < pDestArrival->cost ||
                (nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) {
-               memmove(&arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n + 1],
-                  &arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n],
-                  sizeof(lzsa_arrival) * (NMATCHES_PER_ARRIVAL_V1 - n - 1));
+               memmove(&arrival[((i + 1) << ARRIVALS_PER_POSITION_SHIFT) + n + 1],
+                  &arrival[((i + 1) << ARRIVALS_PER_POSITION_SHIFT) + n],
+                  sizeof(lzsa_arrival) * (NARRIVALS_PER_POSITION_V1 - n - 1));
 
                pDestArrival->cost = nCodingChoiceCost;
                pDestArrival->from_pos = i;
                pDestArrival->from_slot = j + 1;
-               pDestArrival->match_offset = 0;
                pDestArrival->match_len = 0;
                pDestArrival->num_literals = nNumLiterals;
                pDestArrival->score = nScore;
-               pDestArrival->rep_offset = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].rep_offset;
+               pDestArrival->rep_offset = cur_arrival[j].rep_offset;
                break;
             }
          }
       }
 
       const lzsa_match *match = pCompressor->match + ((i - nStartOffset) << MATCHES_PER_INDEX_SHIFT_V1);
+      int nNumArrivalsForThisPos = j;
 
       for (m = 0; m < NMATCHES_PER_INDEX_V1 && match[m].length; m++) {
          int nMatchLen = match[m].length;
          int nMatchOffsetCost = lzsa_get_offset_cost_v1(match[m].offset);
          int nStartingMatchLen, k;
 
-         if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS))
-            nMatchLen = nEndOffset - LAST_LITERALS - i;
+         if ((i + nMatchLen) > nEndOffset)
+            nMatchLen = nEndOffset - i;
 
          if (nMatchLen >= LEAVE_ALONE_MATCH_SIZE)
             nStartingMatchLen = nMatchLen;
@@ -225,43 +228,48 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
          for (k = nStartingMatchLen; k <= nMatchLen; k++) {
             int nMatchLenCost = lzsa_get_match_varlen_size_v1(k - MIN_MATCH_SIZE_V1);
 
-            for (j = 0; j < NMATCHES_PER_ARRIVAL_V1 && arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].from_slot; j++) {
-               int nPrevCost = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].cost;
+            lzsa_arrival *pDestSlots = &arrival[(i + k) << ARRIVALS_PER_POSITION_SHIFT];
+
+            for (j = 0; j < nNumArrivalsForThisPos; j++) {
+               int nPrevCost = cur_arrival[j].cost;
                int nCodingChoiceCost = nPrevCost + 8 /* token */ /* the actual cost of the literals themselves accumulates up the chain */ + nMatchOffsetCost + nMatchLenCost;
-               int nScore = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].score + 5;
                int exists = 0;
 
-               if (!nFavorRatio && !arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].num_literals)
-                  nCodingChoiceCost += MODESWITCH_PENALTY;
+               if (!cur_arrival[j].num_literals)
+                  nCodingChoiceCost += nModeSwitchPenalty;
 
                for (n = 0;
-                  n < NMATCHES_PER_ARRIVAL_V1 && arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].from_slot && arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].cost <= nCodingChoiceCost;
+                  n < NARRIVALS_PER_POSITION_V1 && pDestSlots[n].from_slot && pDestSlots[n].cost <= nCodingChoiceCost;
                   n++) {
-                  if (lzsa_get_offset_cost_v1(arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].rep_offset) == lzsa_get_offset_cost_v1(match[m].offset)) {
+                  if (lzsa_get_offset_cost_v1(pDestSlots[n].rep_offset) == nMatchOffsetCost) {
                      exists = 1;
                      break;
                   }
                }
 
-               for (n = 0; !exists && n < NMATCHES_PER_ARRIVAL_V1 /* we only need the literals + short match cost + long match cost cases */; n++) {
-                  lzsa_arrival *pDestArrival = &arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n];
+               if (!exists) {
+                  int nScore = cur_arrival[j].score + 5;
 
-                  if (pDestArrival->from_slot == 0 ||
-                     nCodingChoiceCost < pDestArrival->cost ||
-                     (nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) {
-                     memmove(&arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n + 1],
-                        &arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n],
-                        sizeof(lzsa_arrival) * (NMATCHES_PER_ARRIVAL_V1 - n - 1));
+                  for (n = 0; n < NARRIVALS_PER_POSITION_V1 /* we only need the literals + short match cost + long match cost cases */; n++) {
+                     lzsa_arrival *pDestArrival = &pDestSlots[n];
 
-                     pDestArrival->cost = nCodingChoiceCost;
-                     pDestArrival->from_pos = i;
-                     pDestArrival->from_slot = j + 1;
-                     pDestArrival->match_offset = match[m].offset;
-                     pDestArrival->match_len = k;
-                     pDestArrival->num_literals = 0;
-                     pDestArrival->score = nScore;
-                     pDestArrival->rep_offset = match[m].offset;
-                     break;
+                     if (pDestArrival->from_slot == 0 ||
+                        nCodingChoiceCost < pDestArrival->cost ||
+                        (nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) {
+                        memmove(&pDestSlots[n + 1],
+                           &pDestSlots[n],
+                           sizeof(lzsa_arrival) * (NARRIVALS_PER_POSITION_V1 - n - 1));
+
+                        pDestArrival->cost = nCodingChoiceCost;
+                        pDestArrival->from_pos = i;
+                        pDestArrival->from_slot = j + 1;
+                        pDestArrival->match_len = k;
+                        pDestArrival->num_literals = 0;
+                        pDestArrival->score = nScore;
+                        pDestArrival->rep_offset = match[m].offset;
+                        j = NARRIVALS_PER_POSITION_V1;
+                        break;
+                     }
                   }
                }
             }
@@ -269,14 +277,17 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
       }
    }
 
-   lzsa_arrival *end_arrival = &arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + 0];
+   lzsa_arrival *end_arrival = &arrival[(i << ARRIVALS_PER_POSITION_SHIFT) + 0];
 
    while (end_arrival->from_slot > 0 && end_arrival->from_pos >= 0) {
       if (end_arrival->from_pos >= nEndOffset) return;
       pBestMatch[end_arrival->from_pos].length = end_arrival->match_len;
-      pBestMatch[end_arrival->from_pos].offset = end_arrival->match_offset;
+      if (end_arrival->match_len)
+         pBestMatch[end_arrival->from_pos].offset = end_arrival->rep_offset;
+      else
+         pBestMatch[end_arrival->from_pos].offset = 0;
 
-      end_arrival = &arrival[(end_arrival->from_pos << MATCHES_PER_ARRIVAL_SHIFT) + (end_arrival->from_slot - 1)];
+      end_arrival = &arrival[(end_arrival->from_pos << ARRIVALS_PER_POSITION_SHIFT) + (end_arrival->from_slot - 1)];
    }
 }
 
@@ -301,12 +312,12 @@ static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const un
       lzsa_match *pMatch = pBestMatch + i;
 
       if (pMatch->length == 0 &&
-         (i + 1) < (nEndOffset - LAST_LITERALS) &&
+         (i + 1) < nEndOffset &&
          pBestMatch[i + 1].length >= MIN_MATCH_SIZE_V1 &&
          pBestMatch[i + 1].length < MAX_VARLEN &&
          pBestMatch[i + 1].offset &&
          i >= pBestMatch[i + 1].offset &&
-         (i + pBestMatch[i + 1].length + 1) <= (nEndOffset - LAST_LITERALS) &&
+         (i + pBestMatch[i + 1].length + 1) <= nEndOffset &&
          !memcmp(pInWindow + i - (pBestMatch[i + 1].offset), pInWindow + i, pBestMatch[i + 1].length + 1)) {
          int nCurLenSize = lzsa_get_match_varlen_size_v1(pBestMatch[i + 1].length - MIN_MATCH_SIZE_V1);
          int nReducedLenSize = lzsa_get_match_varlen_size_v1(pBestMatch[i + 1].length + 1 - MIN_MATCH_SIZE_V1);

src/shrink_context.c

@@ -62,6 +62,9 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
    pCompressor->best_match = NULL;
    pCompressor->improved_match = NULL;
    pCompressor->arrival = NULL;
+   pCompressor->rep_handled_mask = NULL;
+   pCompressor->first_offset_for_byte = NULL;
+   pCompressor->next_offset_for_pos = NULL;
    pCompressor->min_match_size = nMinMatchSize;
    if (pCompressor->min_match_size < nMinMatchSizeForFormat)
       pCompressor->min_match_size = nMinMatchSizeForFormat;
@@ -89,7 +92,7 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
             pCompressor->open_intervals = (unsigned int *)malloc((LCP_AND_TAG_MAX + 1) * sizeof(unsigned int));
 
             if (pCompressor->open_intervals) {
-               pCompressor->arrival = (lzsa_arrival *)malloc(((BLOCK_SIZE + 1) << MATCHES_PER_ARRIVAL_SHIFT) * sizeof(lzsa_arrival));
+               pCompressor->arrival = (lzsa_arrival *)malloc(((BLOCK_SIZE + 1) << ARRIVALS_PER_POSITION_SHIFT) * sizeof(lzsa_arrival));
    
                if (pCompressor->arrival) {
                   pCompressor->best_match = (lzsa_match *)malloc(BLOCK_SIZE * sizeof(lzsa_match));
@@ -102,8 +105,23 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
                            pCompressor->match = (lzsa_match *)malloc(BLOCK_SIZE * NMATCHES_PER_INDEX_V2 * sizeof(lzsa_match));
                         else
                            pCompressor->match = (lzsa_match *)malloc(BLOCK_SIZE * NMATCHES_PER_INDEX_V1 * sizeof(lzsa_match));
-                        if (pCompressor->match)
-                           return 0;
+                        if (pCompressor->match) {
+                           if (pCompressor->format_version == 2) {
+                              pCompressor->rep_handled_mask = (char*)malloc(NARRIVALS_PER_POSITION_V2_BIG * ((LCP_MAX + 1) / 8) * sizeof(char));
+                              if (pCompressor->rep_handled_mask) {
+                                 pCompressor->first_offset_for_byte = (int*)malloc(65536 * sizeof(int));
+                                 if (pCompressor->first_offset_for_byte) {
+                                    pCompressor->next_offset_for_pos = (int*)malloc(BLOCK_SIZE * sizeof(int));
+                                    if (pCompressor->next_offset_for_pos) {
+                                       return 0;
+                                    }
+                                 }
+                              }
+                           }
+                           else {
+                              return 0;
+                           }
+                        }
                      }
                   }
                }
@@ -124,6 +142,21 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
 void lzsa_compressor_destroy(lzsa_compressor *pCompressor) {
    divsufsort_destroy(&pCompressor->divsufsort_context);
 
+   if (pCompressor->next_offset_for_pos) {
+      free(pCompressor->next_offset_for_pos);
+      pCompressor->next_offset_for_pos = NULL;
+   }
+
+   if (pCompressor->first_offset_for_byte) {
+      free(pCompressor->first_offset_for_byte);
+      pCompressor->first_offset_for_byte = NULL;
+   }
+
+   if (pCompressor->rep_handled_mask) {
+      free(pCompressor->rep_handled_mask);
+      pCompressor->rep_handled_mask = NULL;
+   }
+
    if (pCompressor->match) {
       free(pCompressor->match);
       pCompressor->match = NULL;

src/shrink_context.h

@@ -49,10 +49,10 @@ extern "C" {
 #define VISITED_FLAG 0x80000000
 #define EXCL_VISITED_MASK  0x7fffffff
 
-#define NMATCHES_PER_ARRIVAL_V1 8
-#define NMATCHES_PER_ARRIVAL_V2_SMALL 9
-#define NMATCHES_PER_ARRIVAL_V2_BIG 32
-#define MATCHES_PER_ARRIVAL_SHIFT 5
+#define NARRIVALS_PER_POSITION_V1 8
+#define NARRIVALS_PER_POSITION_V2_SMALL 9
+#define NARRIVALS_PER_POSITION_V2_BIG 32
+#define ARRIVALS_PER_POSITION_SHIFT 5
 
 #define NMATCHES_PER_INDEX_V1 8
 #define MATCHES_PER_INDEX_SHIFT_V1 3
@@ -63,8 +63,6 @@ extern "C" {
 #define LEAVE_ALONE_MATCH_SIZE 300
 #define LEAVE_ALONE_MATCH_SIZE_SMALL 1000
 
-#define LAST_LITERALS 0
-
 #define MODESWITCH_PENALTY 3
 
 /** One match */
@@ -81,12 +79,10 @@ typedef struct {
 
    int from_pos;
    unsigned short rep_len;
+   unsigned short match_len;
    int rep_pos;
    int num_literals;
    int score;
-
-   unsigned short match_offset;
-   unsigned short match_len;
 } lzsa_arrival;
 
 /** Compression statistics */
@@ -128,6 +124,9 @@ typedef struct _lzsa_compressor {
    lzsa_match *best_match;
    lzsa_match *improved_match;
    lzsa_arrival *arrival;
+   char *rep_handled_mask;
+   int *first_offset_for_byte;
+   int *next_offset_for_pos;
    int min_match_size;
    int format_version;
    int flags;