mirror of
https://github.com/emmanuel-marty/lzsa.git
synced 2024-06-01 06:41:43 +00:00
Compare commits
24 Commits
Author | SHA1 | Date | |
---|---|---|---|
|
15ee2dfe11 | ||
|
35ec6d73da | ||
|
6b08bc3990 | ||
|
9350d977bf | ||
|
82f03b55e3 | ||
|
583e4db62e | ||
|
398885a52d | ||
|
21a0dc70c8 | ||
|
8cea101625 | ||
|
b86ccf8f7b | ||
|
185ea0cbf2 | ||
|
ed81dd69df | ||
|
bea90736d5 | ||
|
3eaf926c1a | ||
|
1bca5b995a | ||
|
5484395465 | ||
|
34ed06abfb | ||
|
930383a18a | ||
|
eeec526eeb | ||
|
613f3ef0d7 | ||
|
38bfea7ecf | ||
|
a5f3691d4f | ||
|
55101e8ac7 | ||
|
b1a87b55da |
|
@ -21,6 +21,8 @@ The [desolate](https://github.com/nzeemin/spectrum-desolate) game port to the ZX
|
|||
|
||||
The [Lowtech demo](https://github.com/wiz21b/lowtech) for the Apple II+ and IIe, by Wiz/Imphobia, compresses data with LZSA.
|
||||
|
||||
The [Druid & Droid](https://leosoft.itch.io/druid-and-droid) game for the Amstrad CPC, also uses LZSA for compression.
|
||||
|
||||
The LZSA compression tool uses an aggressive optimal packing strategy to try to find the sequence of commands that gives the smallest packed file that decompresses to the original while maintaining the maximum possible decompression speed.
|
||||
|
||||
The compression formats give the user choices that range from decompressing faster than LZ4 on 8-bit systems with better compression, to compressing as well as ZX7 with much better decompression speed. LZSA1 is designed to replace LZ4 and LZSA2 to replace ZX7, in 8-bit scenarios.
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
;
|
||||
; Speed-optimized LZSA2 decompressor by spke & uniabis (216 bytes)
|
||||
; Speed-optimized LZSA2 decompressor by spke & uniabis (210 bytes)
|
||||
;
|
||||
; ver.00 by spke for LZSA 1.0.0 (02-07/06/2019, 218 bytes);
|
||||
; ver.01 by spke for LZSA 1.0.5 (24/07/2019, added support for backward decompression);
|
||||
|
@ -8,7 +8,8 @@
|
|||
; ver.04 by spke for LZSA 1.0.7 (01/08/2019, 214(+1) bytes, +0.2% speed and small re-organization of macros);
|
||||
; ver.05 by spke (27/08/2019, 216(+2) bytes, +1.1% speed);
|
||||
; ver.06 by spke for LZSA 1.1.0 (26/09/2019, added full revision history);
|
||||
; ver.07 by spke for LZSA 1.1.1 (10/10/2019, +0.2% speed and an option for unrolled copying of long matches)
|
||||
; ver.07 by spke for LZSA 1.1.1 (10/10/2019, +0.2% speed and an option for unrolled copying of long matches);
|
||||
; ver.08 by spke (07-08/04/2022, 210(-6) bytes, +1.7% speed, using self-modifying code by default)
|
||||
;
|
||||
; The data must be compressed using the command line compressor by Emmanuel Marty
|
||||
; The compression is done as follows:
|
||||
|
@ -57,8 +58,7 @@
|
|||
; 3. This notice may not be removed or altered from any source distribution.
|
||||
|
||||
; DEFINE UNROLL_LONG_MATCHES ; uncomment for faster decompression of very compressible data (+38 bytes)
|
||||
; DEFINE BACKWARD_DECOMPRESS ; uncomment for data compressed with option -b
|
||||
; DEFINE HD64180 ; uncomment for systems using Hitachi HD64180
|
||||
; DEFINE BACKWARD_DECOMPRESS ; uncomment for data compressed with option -b (+5 bytes, -3.2% speed)
|
||||
|
||||
IFNDEF BACKWARD_DECOMPRESS
|
||||
|
||||
|
@ -67,7 +67,7 @@
|
|||
ENDM
|
||||
|
||||
MACRO ADD_OFFSET
|
||||
ex de,hl : add hl,de
|
||||
add hl,de
|
||||
ENDM
|
||||
|
||||
MACRO COPY1
|
||||
|
@ -85,8 +85,9 @@
|
|||
ENDM
|
||||
|
||||
MACRO ADD_OFFSET
|
||||
ex de,hl : ld a,e : sub l : ld l,a
|
||||
ld a,d : sbc h : ld h,a ; 4*4+3*4 = 28t / 7 bytes
|
||||
; HL = DE - HL
|
||||
ld a,e : sub l : ld l,a
|
||||
ld a,d : sbc h : ld h,a ; 6*4 = 24t / 6 bytes
|
||||
ENDM
|
||||
|
||||
MACRO COPY1
|
||||
|
@ -99,37 +100,15 @@
|
|||
|
||||
ENDIF
|
||||
|
||||
IFNDEF HD64180
|
||||
|
||||
MACRO LD_IX_DE
|
||||
ld ixl,e : ld ixh,d
|
||||
ENDM
|
||||
|
||||
MACRO LD_DE_IX
|
||||
ld e,ixl : ld d,ixh
|
||||
ENDM
|
||||
|
||||
ELSE
|
||||
|
||||
MACRO LD_IX_DE
|
||||
push de : pop ix
|
||||
ENDM
|
||||
|
||||
MACRO LD_DE_IX
|
||||
push ix : pop de
|
||||
ENDM
|
||||
|
||||
ENDIF
|
||||
|
||||
@DecompressLZSA2:
|
||||
; A' stores next nibble as %1111.... or assumed to contain trash
|
||||
; B is assumed to be 0
|
||||
; B is assumed to be 0 in many places
|
||||
ld b,0 : scf : exa : jr ReadToken
|
||||
|
||||
|
||||
|
||||
|
||||
ManyLiterals: ld a,18 : add (hl) : NEXT_HL : jr nc,CopyLiterals
|
||||
ManyLiterals: ld a,18 : add (hl) : NEXT_HL : jr nc,CopyMoreLiterals
|
||||
ld c,(hl) : NEXT_HL
|
||||
ld a,b : ld b,(hl)
|
||||
jr ReadToken.NEXTHLuseBC
|
||||
|
@ -140,23 +119,31 @@ ManyLiterals: ld a,18 : add (hl) : NEXT_HL : jr nc,CopyLiterals
|
|||
MoreLiterals: ld b,(hl) : NEXT_HL
|
||||
scf : exa : jr nc,.noUpdate
|
||||
|
||||
ld a,(hl) : or #F0 : exa
|
||||
ld a,(hl) : NEXT_HL : or #0F
|
||||
; nibbles are read left-to-right; spare nibbles are kept in AF'
|
||||
; and flag NC indicates that a nibble is available
|
||||
ld a,(hl) : or a : exa
|
||||
ld a,(hl) : NEXT_HL
|
||||
rrca : rrca : rrca : rrca
|
||||
|
||||
.noUpdate ;sub #F0-3 : cp 15+3 : jr z,ManyLiterals
|
||||
.noUpdate or #F0
|
||||
;sub #F0-3 : cp 15+3 : jr z,ManyLiterals
|
||||
inc a : jr z,ManyLiterals : sub #F0-3+1
|
||||
|
||||
CopyLiterals: ld c,a : ld a,b : ld b,0
|
||||
COPYBC
|
||||
push de : or a : jp p,CASE0xx ;: jr CASE1xx
|
||||
CopyMoreLiterals: ld c,a : ld a,b : ld b,0
|
||||
COPY1
|
||||
COPY1
|
||||
COPYBC
|
||||
|
||||
or a : jp p,CASE0xx
|
||||
|
||||
cp %11000000 : jr c,CASE10x
|
||||
|
||||
CASE11x cp %11100000 : jr c,CASE110
|
||||
|
||||
; "111": repeated offset
|
||||
CASE111: LD_DE_IX : jr MatchLen
|
||||
CASE11x cp %11100000 : jr nc,MatchLen
|
||||
|
||||
; "110": 16-bit offset
|
||||
CASE110: ld b,(hl) : NEXT_HL : jr ReadOffsetC
|
||||
|
||||
|
||||
|
||||
|
||||
|
@ -166,69 +153,66 @@ Literals0011: jr nz,MoreLiterals
|
|||
; if "LL" of the byte token is equal to 0,
|
||||
; there are no literals to copy
|
||||
NoLiterals: or (hl) : NEXT_HL
|
||||
push de : jp m,CASE1xx
|
||||
jp m,CASE1xx
|
||||
|
||||
; short (5 or 9 bit long) offsets
|
||||
CASE0xx ld d,#FF : cp %01000000 : jr c,CASE00x
|
||||
CASE0xx cp %01000000 : jr c,CASE00x
|
||||
|
||||
; "01x": the case of the 9-bit offset
|
||||
CASE01x: cp %01100000 : rl d
|
||||
; "01x": the case of the 9-bit offset
|
||||
CASE01x: dec b : cp %01100000 : rl b
|
||||
|
||||
ReadOffsetE ld e,(hl) : NEXT_HL
|
||||
ReadOffsetC ld c,(hl) : NEXT_HL
|
||||
|
||||
SaveOffset: LD_IX_DE
|
||||
SaveOffset ld (CopyMatch.PrevOffset),bc : ld b,0
|
||||
|
||||
MatchLen: inc a : and %00000111 : jr z,LongerMatch : inc a
|
||||
MatchLen inc a : and %00000111 : jr z,LongerMatch : inc a
|
||||
|
||||
CopyMatch: ld c,a
|
||||
.useC ex (sp),hl ; BC = len, DE = offset, HL = dest, SP ->[dest,src]
|
||||
ADD_OFFSET ; BC = len, DE = dest, HL = dest-offset, SP->[src]
|
||||
COPY1
|
||||
COPYBC
|
||||
.popSrc pop hl
|
||||
CopyMatch: ld c,a
|
||||
.useC push hl
|
||||
.PrevOffset EQU $+1 : ld hl,0
|
||||
ADD_OFFSET
|
||||
COPY1
|
||||
COPYBC
|
||||
.popSrc pop hl
|
||||
|
||||
; compressed data stream contains records
|
||||
; each record begins with the byte token "XYZ|LL|MMM"
|
||||
ReadToken: ld a,(hl) : and %00011000 : jp pe,Literals0011 ; process the cases 00 and 11 separately
|
||||
|
||||
rrca : rrca : rrca
|
||||
rrca : rrca : rrca
|
||||
|
||||
ld c,a : ld a,(hl) ; token is re-read for further processing
|
||||
.NEXTHLuseBC NEXT_HL
|
||||
COPYBC
|
||||
ld c,a : ld a,(hl) ; token is re-read for further processing
|
||||
.NEXTHLuseBC NEXT_HL
|
||||
COPYBC
|
||||
|
||||
; the token and literals are followed by the offset
|
||||
push de : or a : jp p,CASE0xx
|
||||
or a : jp p,CASE0xx
|
||||
|
||||
CASE1xx cp %11000000 : jr nc,CASE11x
|
||||
|
||||
; "10x": the case of the 13-bit offset
|
||||
CASE10x: ld c,a : exa : jr nc,.noUpdate
|
||||
|
||||
ld a,(hl) : or #F0 : exa
|
||||
ld a,(hl) : NEXT_HL : or #0F
|
||||
ld a,(hl) : or a : exa
|
||||
ld a,(hl) : NEXT_HL
|
||||
rrca : rrca : rrca : rrca
|
||||
|
||||
.noUpdate ld d,a : ld a,c
|
||||
cp %10100000 : dec d : rl d : jr ReadOffsetE
|
||||
.noUpdate or #F0 : ld b,a : ld a,c
|
||||
cp %10100000 : dec b : rl b : jr ReadOffsetC
|
||||
|
||||
|
||||
|
||||
; "110": 16-bit offset
|
||||
CASE110: ld d,(hl) : NEXT_HL : jr ReadOffsetE
|
||||
|
||||
|
||||
|
||||
|
||||
; "00x": the case of the 5-bit offset
|
||||
CASE00x: ld c,a : exa : jr nc,.noUpdate
|
||||
CASE00x: ld b,a : exa : jr nc,.noUpdate
|
||||
|
||||
ld a,(hl) : or #F0 : exa
|
||||
ld a,(hl) : NEXT_HL : or #0F
|
||||
ld a,(hl) : or a : exa
|
||||
ld a,(hl) : NEXT_HL
|
||||
rrca : rrca : rrca : rrca
|
||||
|
||||
.noUpdate ld e,a : ld a,c
|
||||
cp %00100000 : rl e : jp SaveOffset
|
||||
.noUpdate or #F0 : ld c,a : ld a,b
|
||||
cp %00100000 : rl c
|
||||
ld b,#FF : jr SaveOffset
|
||||
|
||||
|
||||
|
||||
|
@ -236,27 +220,27 @@ CASE00x: ld c,a : exa : jr nc,.noUpdate
|
|||
|
||||
LongerMatch: scf : exa : jr nc,.noUpdate
|
||||
|
||||
ld a,(hl) : or #F0 : exa
|
||||
ld a,(hl) : NEXT_HL : or #0F
|
||||
ld a,(hl) : or a : exa
|
||||
ld a,(hl) : NEXT_HL
|
||||
rrca : rrca : rrca : rrca
|
||||
|
||||
.noUpdate sub #F0-9 : cp 15+9 : jr c,CopyMatch
|
||||
.noUpdate or #F0 : sub #F0-9 : cp 15+9 : jr c,CopyMatch
|
||||
|
||||
IFNDEF UNROLL_LONG_MATCHES
|
||||
|
||||
LongMatch: add (hl) : NEXT_HL : jr nc,CopyMatch
|
||||
ld c,(hl) : NEXT_HL
|
||||
ld b,(hl) : NEXT_HL : jr nz,CopyMatch.useC
|
||||
pop de : ret
|
||||
ret
|
||||
|
||||
ELSE
|
||||
|
||||
LongMatch: add (hl) : NEXT_HL : jr c,VeryLongMatch
|
||||
|
||||
ld c,a
|
||||
.useC ex (sp),hl
|
||||
.useC push hl
|
||||
ld hl,(CopyMatch.PrevOffset)
|
||||
ADD_OFFSET
|
||||
COPY1
|
||||
|
||||
; this is an unrolled equivalent of LDIR
|
||||
xor a : sub c
|
||||
|
@ -271,7 +255,7 @@ LongMatch: add (hl) : NEXT_HL : jr c,VeryLongMatch
|
|||
|
||||
VeryLongMatch: ld c,(hl) : NEXT_HL
|
||||
ld b,(hl) : NEXT_HL : jr nz,LongMatch.useC
|
||||
pop de : ret
|
||||
ret
|
||||
|
||||
ENDIF
|
||||
|
||||
|
|
|
@ -30,8 +30,10 @@
|
|||
*
|
||||
*/
|
||||
|
||||
#define _POSIX_C_SOURCE 200808
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/types.h>
|
||||
#include "format.h"
|
||||
#include "lib.h"
|
||||
|
||||
|
|
|
@ -120,7 +120,7 @@ static inline FORCE_INLINE int lzsa_build_match_len_v1(const unsigned char **ppI
|
|||
*
|
||||
* @return size of decompressed data in bytes, or -1 for error
|
||||
*/
|
||||
int lzsa_decompressor_expand_block_v1(const unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize) {
|
||||
int lzsa_decompressor_expand_block_v1(const unsigned char *pInBlock, const int nBlockSize, unsigned char *pOutData, const int nOutDataOffset, const int nBlockMaxSize) {
|
||||
const unsigned char *pInBlockEnd = pInBlock + nBlockSize;
|
||||
unsigned char *pCurOutData = pOutData + nOutDataOffset;
|
||||
const unsigned char *pOutDataEnd = pCurOutData + nBlockMaxSize;
|
||||
|
|
|
@ -44,6 +44,6 @@
|
|||
*
|
||||
* @return size of decompressed data in bytes, or -1 for error
|
||||
*/
|
||||
int lzsa_decompressor_expand_block_v1(const unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize);
|
||||
int lzsa_decompressor_expand_block_v1(const unsigned char *pInBlock, const int nBlockSize, unsigned char *pOutData, const int nOutDataOffset, const int nBlockMaxSize);
|
||||
|
||||
#endif /* _EXPAND_BLOCK_V1_H */
|
||||
|
|
|
@ -109,7 +109,7 @@ static inline FORCE_INLINE int lzsa_build_len_v2(const unsigned char **ppInBlock
|
|||
*
|
||||
* @return size of decompressed data in bytes, or -1 for error
|
||||
*/
|
||||
int lzsa_decompressor_expand_block_v2(const unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize) {
|
||||
int lzsa_decompressor_expand_block_v2(const unsigned char *pInBlock, const int nBlockSize, unsigned char *pOutData, const int nOutDataOffset, const int nBlockMaxSize) {
|
||||
const unsigned char *pInBlockEnd = pInBlock + nBlockSize;
|
||||
unsigned char *pCurOutData = pOutData + nOutDataOffset;
|
||||
const unsigned char *pOutDataEnd = pCurOutData + nBlockMaxSize;
|
||||
|
@ -147,7 +147,7 @@ int lzsa_decompressor_expand_block_v2(const unsigned char *pInBlock, int nBlockS
|
|||
}
|
||||
|
||||
if (pInBlock < pInBlockEnd) { /* The last token in the block does not include match information */
|
||||
unsigned char nOffsetMode = token & 0xc0;
|
||||
const unsigned char nOffsetMode = token & 0xc0;
|
||||
unsigned int nValue;
|
||||
|
||||
switch (nOffsetMode) {
|
||||
|
|
|
@ -44,6 +44,6 @@
|
|||
*
|
||||
* @return size of decompressed data in bytes, or -1 for error
|
||||
*/
|
||||
int lzsa_decompressor_expand_block_v2(const unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize);
|
||||
int lzsa_decompressor_expand_block_v2(const unsigned char *pInBlock, const int nBlockSize, unsigned char *pOutData, const int nOutDataOffset, const int nBlockMaxSize);
|
||||
|
||||
#endif /* _EXPAND_BLOCK_V2_H */
|
||||
|
|
|
@ -50,7 +50,7 @@
|
|||
*
|
||||
* @return size of decompressed data in bytes, or -1 for error
|
||||
*/
|
||||
int lzsa_decompressor_expand_block(unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize, const int nFormatVersion, const int nFlags) {
|
||||
int lzsa_decompressor_expand_block(unsigned char *pInBlock, const int nBlockSize, unsigned char *pOutData, const int nOutDataOffset, const int nBlockMaxSize, const int nFormatVersion, const int nFlags) {
|
||||
int nDecompressedSize;
|
||||
|
||||
if (nFlags & LZSA_FLAG_RAW_BACKWARD) {
|
||||
|
|
|
@ -52,7 +52,7 @@ extern "C" {
|
|||
*
|
||||
* @return size of decompressed data in bytes, or -1 for error
|
||||
*/
|
||||
int lzsa_decompressor_expand_block(unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize, const int nFormatVersion, const int nFlags);
|
||||
int lzsa_decompressor_expand_block(unsigned char *pInBlock, const int nBlockSize, unsigned char *pOutData, const int nOutDataOffset, const int nBlockMaxSize, const int nFormatVersion, const int nFlags);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -39,9 +39,6 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Forward declaration */
|
||||
typedef enum _lzsa_status_t lzsa_status_t;
|
||||
|
||||
/*-------------- File API -------------- */
|
||||
|
||||
/**
|
||||
|
|
|
@ -60,10 +60,11 @@ int lzsa_get_frame_size(void) {
|
|||
*
|
||||
* @param pFrameData encoding buffer
|
||||
* @param nMaxFrameDataSize max encoding buffer size, in bytes
|
||||
* @param nFormatVersion version of format to use (1-2)
|
||||
*
|
||||
* @return number of encoded bytes, or -1 for failure
|
||||
*/
|
||||
int lzsa_encode_header(unsigned char *pFrameData, const int nMaxFrameDataSize, int nFormatVersion) {
|
||||
int lzsa_encode_header(unsigned char *pFrameData, const int nMaxFrameDataSize, const int nFormatVersion) {
|
||||
if (nMaxFrameDataSize >= 3 && (nFormatVersion == 1 || nFormatVersion == 2)) {
|
||||
pFrameData[0] = LZSA_ID_0; /* Magic number */
|
||||
pFrameData[1] = LZSA_ID_1;
|
||||
|
@ -146,6 +147,7 @@ int lzsa_encode_footer_frame(unsigned char *pFrameData, const int nMaxFrameDataS
|
|||
*
|
||||
* @param pFrameData data bytes
|
||||
* @param nFrameDataSize number of bytes to decode
|
||||
* @param nFormatVersion pointer to returned format version, if successful
|
||||
*
|
||||
* @return 0 for success, or -1 for failure
|
||||
*/
|
||||
|
|
|
@ -56,10 +56,11 @@ int lzsa_get_frame_size(void);
|
|||
*
|
||||
* @param pFrameData encoding buffer
|
||||
* @param nMaxFrameDataSize max encoding buffer size, in bytes
|
||||
* @param nFormatVersion version of format to use (1-2)
|
||||
*
|
||||
* @return number of encoded bytes, or -1 for failure
|
||||
*/
|
||||
int lzsa_encode_header(unsigned char *pFrameData, const int nMaxFrameDataSize, int nFormatVersion);
|
||||
int lzsa_encode_header(unsigned char *pFrameData, const int nMaxFrameDataSize, const int nFormatVersion);
|
||||
|
||||
/**
|
||||
* Encode compressed block frame header
|
||||
|
@ -98,6 +99,7 @@ int lzsa_encode_footer_frame(unsigned char *pFrameData, const int nMaxFrameDataS
|
|||
*
|
||||
* @param pFrameData data bytes
|
||||
* @param nFrameDataSize number of bytes to decode
|
||||
* @param nFormatVersion pointer to returned format version, if successful
|
||||
*
|
||||
* @return 0 for success, or -1 for failure
|
||||
*/
|
||||
|
|
22
src/lib.h
22
src/lib.h
|
@ -48,24 +48,6 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
/** High level status for compression and decompression */
|
||||
typedef enum _lzsa_status_t {
|
||||
LZSA_OK = 0, /**< Success */
|
||||
LZSA_ERROR_SRC, /**< Error reading input */
|
||||
LZSA_ERROR_DST, /**< Error reading output */
|
||||
LZSA_ERROR_DICTIONARY, /**< Error reading dictionary */
|
||||
LZSA_ERROR_MEMORY, /**< Out of memory */
|
||||
|
||||
/* Compression-specific status codes */
|
||||
LZSA_ERROR_COMPRESSION, /**< Internal compression error */
|
||||
LZSA_ERROR_RAW_TOOLARGE, /**< Input is too large to be compressed to a raw block */
|
||||
LZSA_ERROR_RAW_UNCOMPRESSED, /**< Input is incompressible and raw blocks don't support uncompressed data */
|
||||
|
||||
/* Decompression-specific status codes */
|
||||
LZSA_ERROR_FORMAT, /**< Invalid input format or magic number when decompressing */
|
||||
LZSA_ERROR_DECOMPRESSION /**< Internal decompression error */
|
||||
} lzsa_status_t;
|
||||
|
||||
/* Compression flags */
|
||||
#define LZSA_FLAG_FAVOR_RATIO (1<<0) /**< 1 to compress with the best ratio, 0 to trade some compression ratio for extra decompression speed */
|
||||
#define LZSA_FLAG_RAW_BLOCK (1<<1) /**< 1 to emit raw block */
|
||||
|
@ -78,11 +60,11 @@ typedef enum _lzsa_status_t {
|
|||
* @param nBufferSize size of buffer in bytes
|
||||
*/
|
||||
static inline void lzsa_reverse_buffer(unsigned char *pBuffer, const int nBufferSize) {
|
||||
int nMidPoint = nBufferSize / 2;
|
||||
const int nMidPoint = nBufferSize / 2;
|
||||
int i, j;
|
||||
|
||||
for (i = 0, j = nBufferSize - 1; i < nMidPoint; i++, j--) {
|
||||
unsigned char c = pBuffer[i];
|
||||
const unsigned char c = pBuffer[i];
|
||||
pBuffer[i] = pBuffer[j];
|
||||
pBuffer[j] = c;
|
||||
}
|
||||
|
|
|
@ -75,8 +75,8 @@ void divsufsort_destroy(divsufsort_ctx_t *ctx);
|
|||
/**
|
||||
* Constructs the suffix array of a given string.
|
||||
* @param ctx suffix array context
|
||||
* @param T[0..n-1] The input string.
|
||||
* @param SA[0..n-1] The output array of suffixes.
|
||||
* @param T The input string.
|
||||
* @param SA The output array of suffixes.
|
||||
* @param n The length of the given string.
|
||||
* @return 0 if no error occurred, -1 or -2 otherwise.
|
||||
*/
|
||||
|
|
68
src/lzsa.c
68
src/lzsa.c
|
@ -47,7 +47,7 @@
|
|||
#define OPT_RAW_BACKWARD 8
|
||||
#define OPT_STATS 16
|
||||
|
||||
#define TOOL_VERSION "1.3.11"
|
||||
#define TOOL_VERSION "1.4.1"
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
|
@ -120,7 +120,7 @@ static int do_compress(const char *pszInFilename, const char *pszOutFilename, co
|
|||
|
||||
nStatus = lzsa_compress_file(pszInFilename, pszOutFilename, pszDictionaryFilename, nFlags, nMinMatchSize, nFormatVersion, compression_progress, &nOriginalSize, &nCompressedSize, &nCommandCount, &nSafeDist, &stats);
|
||||
|
||||
if ((nOptions & OPT_VERBOSE)) {
|
||||
if (nOptions & OPT_VERBOSE) {
|
||||
nEndTime = do_get_time();
|
||||
}
|
||||
|
||||
|
@ -139,7 +139,7 @@ static int do_compress(const char *pszInFilename, const char *pszOutFilename, co
|
|||
if (nStatus)
|
||||
return 100;
|
||||
|
||||
if ((nOptions & OPT_VERBOSE)) {
|
||||
if (nOptions & OPT_VERBOSE) {
|
||||
double fDelta = ((double)(nEndTime - nStartTime)) / 1000000.0;
|
||||
double fSpeed = ((double)nOriginalSize / 1048576.0) / fDelta;
|
||||
fprintf(stdout, "\rCompressed '%s' in %g seconds, %.02g Mb/s, %d tokens (%g bytes/token), %lld into %lld bytes ==> %g %%\n",
|
||||
|
@ -383,13 +383,13 @@ static int do_compare(const char *pszInFilename, const char *pszOutFilename, con
|
|||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
static void generate_compressible_data(unsigned char *pBuffer, size_t nBufferSize, int nMinMatchSize, unsigned int nSeed, int nNumLiteralValues, float fMatchProbability) {
|
||||
static void generate_compressible_data(unsigned char *pBuffer, const size_t nBufferSize, const int nMinMatchSize, const unsigned int nSeed, const int nNumLiteralValues, const float fMatchProbability) {
|
||||
size_t nIndex = 0;
|
||||
int nMatchProbability = (int)(fMatchProbability * 1023.0f);
|
||||
const int nMatchProbability = (int)(fMatchProbability * 1023.0f);
|
||||
|
||||
srand(nSeed);
|
||||
|
||||
if (nIndex >= nBufferSize) return;
|
||||
if (nBufferSize == 0) return;
|
||||
pBuffer[nIndex++] = rand() % nNumLiteralValues;
|
||||
|
||||
while (nIndex < nBufferSize) {
|
||||
|
@ -423,14 +423,12 @@ static void generate_compressible_data(unsigned char *pBuffer, size_t nBufferSiz
|
|||
}
|
||||
}
|
||||
|
||||
static void xor_data(unsigned char *pBuffer, size_t nBufferSize, unsigned int nSeed, float fXorProbability) {
|
||||
static void xor_data(unsigned char *pBuffer, const size_t nBufferSize, const unsigned int nSeed, const float fXorProbability) {
|
||||
size_t nIndex = 0;
|
||||
int nXorProbability = (int)(fXorProbability * 1023.0f);
|
||||
const int nXorProbability = (const int)(fXorProbability * 1023.0f);
|
||||
|
||||
srand(nSeed);
|
||||
|
||||
if (nIndex >= nBufferSize) return;
|
||||
|
||||
while (nIndex < nBufferSize) {
|
||||
if ((rand() & 1023) < nXorProbability) {
|
||||
pBuffer[nIndex] ^= 0xff;
|
||||
|
@ -439,7 +437,7 @@ static void xor_data(unsigned char *pBuffer, size_t nBufferSize, unsigned int nS
|
|||
}
|
||||
}
|
||||
|
||||
static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, int nFormatVersion) {
|
||||
static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, const int nFormatVersion) {
|
||||
unsigned char *pGeneratedData;
|
||||
unsigned char *pCompressedData;
|
||||
unsigned char *pTmpCompressedData;
|
||||
|
@ -470,7 +468,7 @@ static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, in
|
|||
free(pGeneratedData);
|
||||
pGeneratedData = NULL;
|
||||
|
||||
fprintf(stderr, "out of memory, %zd bytes needed\n", nMaxCompressedDataSize);
|
||||
fprintf(stderr, "out of memory, %zu bytes needed\n", nMaxCompressedDataSize);
|
||||
return 100;
|
||||
}
|
||||
|
||||
|
@ -481,7 +479,7 @@ static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, in
|
|||
free(pGeneratedData);
|
||||
pGeneratedData = NULL;
|
||||
|
||||
fprintf(stderr, "out of memory, %zd bytes needed\n", nMaxCompressedDataSize);
|
||||
fprintf(stderr, "out of memory, %zu bytes needed\n", nMaxCompressedDataSize);
|
||||
return 100;
|
||||
}
|
||||
|
||||
|
@ -514,7 +512,7 @@ static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, in
|
|||
for (nGeneratedDataSize = 1024; nGeneratedDataSize <= ((size_t)((nOptions & OPT_RAW) ? BLOCK_SIZE : (4 * BLOCK_SIZE))); nGeneratedDataSize += nDataSizeStep) {
|
||||
float fMatchProbability;
|
||||
|
||||
fprintf(stdout, "size %zd", nGeneratedDataSize);
|
||||
fprintf(stdout, "size %zu", nGeneratedDataSize);
|
||||
for (fMatchProbability = 0; fMatchProbability <= 0.995f; fMatchProbability += fProbabilitySizeStep) {
|
||||
int nNumLiteralValues[12] = { 1, 2, 3, 15, 30, 56, 96, 137, 178, 191, 255, 256 };
|
||||
float fXorProbability;
|
||||
|
@ -529,7 +527,7 @@ static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, in
|
|||
/* Try to compress it, expected to succeed */
|
||||
size_t nActualCompressedSize = lzsa_compress_inmem(pGeneratedData, pCompressedData, nGeneratedDataSize, lzsa_get_max_compressed_size_inmem(nGeneratedDataSize),
|
||||
nFlags, nMinMatchSize, nFormatVersion);
|
||||
if (nActualCompressedSize == -1 || (int)nActualCompressedSize < (lzsa_get_header_size() + lzsa_get_frame_size() + lzsa_get_frame_size() /* footer */)) {
|
||||
if (nActualCompressedSize == (size_t)-1 || (int)nActualCompressedSize < (lzsa_get_header_size() + lzsa_get_frame_size() + lzsa_get_frame_size() /* footer */)) {
|
||||
free(pTmpDecompressedData);
|
||||
pTmpDecompressedData = NULL;
|
||||
free(pTmpCompressedData);
|
||||
|
@ -539,7 +537,7 @@ static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, in
|
|||
free(pGeneratedData);
|
||||
pGeneratedData = NULL;
|
||||
|
||||
fprintf(stderr, "\nself-test: error compressing size %zd, seed %d, match probability %f, literals range %d\n", nGeneratedDataSize, nSeed, fMatchProbability, nNumLiteralValues[i]);
|
||||
fprintf(stderr, "\nself-test: error compressing size %zu, seed %u, match probability %f, literals range %d\n", nGeneratedDataSize, nSeed, fMatchProbability, nNumLiteralValues[i]);
|
||||
return 100;
|
||||
}
|
||||
|
||||
|
@ -547,7 +545,7 @@ static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, in
|
|||
size_t nActualDecompressedSize;
|
||||
int nDecFormatVersion = nFormatVersion;
|
||||
nActualDecompressedSize = lzsa_decompress_inmem(pCompressedData, pTmpDecompressedData, nActualCompressedSize, nGeneratedDataSize, nFlags, &nDecFormatVersion);
|
||||
if (nActualDecompressedSize == -1) {
|
||||
if (nActualDecompressedSize == (size_t)-1) {
|
||||
free(pTmpDecompressedData);
|
||||
pTmpDecompressedData = NULL;
|
||||
free(pTmpCompressedData);
|
||||
|
@ -557,7 +555,7 @@ static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, in
|
|||
free(pGeneratedData);
|
||||
pGeneratedData = NULL;
|
||||
|
||||
fprintf(stderr, "\nself-test: error decompressing size %zd, seed %d, match probability %f, literals range %d\n", nGeneratedDataSize, nSeed, fMatchProbability, nNumLiteralValues[i]);
|
||||
fprintf(stderr, "\nself-test: error decompressing size %zu, seed %u, match probability %f, literals range %d\n", nGeneratedDataSize, nSeed, fMatchProbability, nNumLiteralValues[i]);
|
||||
return 100;
|
||||
}
|
||||
|
||||
|
@ -571,7 +569,7 @@ static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, in
|
|||
free(pGeneratedData);
|
||||
pGeneratedData = NULL;
|
||||
|
||||
fprintf(stderr, "\nself-test: error comparing decompressed and original data, size %zd, seed %d, match probability %f, literals range %d\n", nGeneratedDataSize, nSeed, fMatchProbability, nNumLiteralValues[i]);
|
||||
fprintf(stderr, "\nself-test: error comparing decompressed and original data, size %zu, seed %u, match probability %f, literals range %d\n", nGeneratedDataSize, nSeed, fMatchProbability, nNumLiteralValues[i]);
|
||||
return 100;
|
||||
}
|
||||
|
||||
|
@ -651,7 +649,7 @@ static int do_compr_benchmark(const char *pszInFilename, const char *pszOutFilen
|
|||
pFileData = (unsigned char*)malloc(nFileSize);
|
||||
if (!pFileData) {
|
||||
fclose(f_in);
|
||||
fprintf(stderr, "out of memory for reading '%s', %zd bytes needed\n", pszInFilename, nFileSize);
|
||||
fprintf(stderr, "out of memory for reading '%s', %zu bytes needed\n", pszInFilename, nFileSize);
|
||||
return 100;
|
||||
}
|
||||
|
||||
|
@ -671,7 +669,7 @@ static int do_compr_benchmark(const char *pszInFilename, const char *pszOutFilen
|
|||
pCompressedData = (unsigned char*)malloc(nMaxCompressedSize + 2048);
|
||||
if (!pCompressedData) {
|
||||
free(pFileData);
|
||||
fprintf(stderr, "out of memory for compressing '%s', %zd bytes needed\n", pszInFilename, nMaxCompressedSize);
|
||||
fprintf(stderr, "out of memory for compressing '%s', %zu bytes needed\n", pszInFilename, nMaxCompressedSize);
|
||||
return 100;
|
||||
}
|
||||
|
||||
|
@ -693,7 +691,7 @@ static int do_compr_benchmark(const char *pszInFilename, const char *pszOutFilen
|
|||
long long t0 = do_get_time();
|
||||
nActualCompressedSize = lzsa_compress_inmem(pFileData, pCompressedData + 1024, nFileSize, nRightGuardPos, nFlags, nMinMatchSize, nFormatVersion);
|
||||
long long t1 = do_get_time();
|
||||
if (nActualCompressedSize == -1) {
|
||||
if (nActualCompressedSize == (size_t)-1) {
|
||||
free(pCompressedData);
|
||||
free(pFileData);
|
||||
fprintf(stderr, "compression error\n");
|
||||
|
@ -701,7 +699,7 @@ static int do_compr_benchmark(const char *pszInFilename, const char *pszOutFilen
|
|||
}
|
||||
|
||||
long long nCurDecTime = t1 - t0;
|
||||
if (nBestCompTime == -1 || nBestCompTime > nCurDecTime)
|
||||
if (nBestCompTime == (size_t)-1 || nBestCompTime > nCurDecTime)
|
||||
nBestCompTime = nCurDecTime;
|
||||
|
||||
/* Check guard bytes before the output buffer */
|
||||
|
@ -742,7 +740,7 @@ static int do_compr_benchmark(const char *pszInFilename, const char *pszOutFilen
|
|||
free(pCompressedData);
|
||||
free(pFileData);
|
||||
|
||||
fprintf(stdout, "compressed size: %zd bytes\n", nActualCompressedSize);
|
||||
fprintf(stdout, "compressed size: %zu bytes\n", nActualCompressedSize);
|
||||
fprintf(stdout, "compression time: %lld microseconds (%g Mb/s)\n", nBestCompTime, ((double)nActualCompressedSize / 1024.0) / ((double)nBestCompTime / 1000.0));
|
||||
|
||||
return 0;
|
||||
|
@ -783,7 +781,7 @@ static int do_dec_benchmark(const char *pszInFilename, const char *pszOutFilenam
|
|||
pFileData = (unsigned char*)malloc(nFileSize);
|
||||
if (!pFileData) {
|
||||
fclose(f_in);
|
||||
fprintf(stderr, "out of memory for reading '%s', %zd bytes needed\n", pszInFilename, nFileSize);
|
||||
fprintf(stderr, "out of memory for reading '%s', %zu bytes needed\n", pszInFilename, nFileSize);
|
||||
return 100;
|
||||
}
|
||||
|
||||
|
@ -802,7 +800,7 @@ static int do_dec_benchmark(const char *pszInFilename, const char *pszOutFilenam
|
|||
nMaxDecompressedSize = 65536;
|
||||
else
|
||||
nMaxDecompressedSize = lzsa_get_max_decompressed_size_inmem(pFileData, nFileSize);
|
||||
if (nMaxDecompressedSize == -1) {
|
||||
if (nMaxDecompressedSize == (size_t)-1) {
|
||||
free(pFileData);
|
||||
fprintf(stderr, "invalid compressed format for file '%s'\n", pszInFilename);
|
||||
return 100;
|
||||
|
@ -811,7 +809,7 @@ static int do_dec_benchmark(const char *pszInFilename, const char *pszOutFilenam
|
|||
pDecompressedData = (unsigned char*)malloc(nMaxDecompressedSize);
|
||||
if (!pDecompressedData) {
|
||||
free(pFileData);
|
||||
fprintf(stderr, "out of memory for decompressing '%s', %zd bytes needed\n", pszInFilename, nMaxDecompressedSize);
|
||||
fprintf(stderr, "out of memory for decompressing '%s', %zu bytes needed\n", pszInFilename, nMaxDecompressedSize);
|
||||
return 100;
|
||||
}
|
||||
|
||||
|
@ -824,7 +822,7 @@ static int do_dec_benchmark(const char *pszInFilename, const char *pszOutFilenam
|
|||
long long t0 = do_get_time();
|
||||
nActualDecompressedSize = lzsa_decompress_inmem(pFileData, pDecompressedData, nFileSize, nMaxDecompressedSize, nFlags, &nFormatVersion);
|
||||
long long t1 = do_get_time();
|
||||
if (nActualDecompressedSize == -1) {
|
||||
if (nActualDecompressedSize == (size_t)-1) {
|
||||
free(pDecompressedData);
|
||||
free(pFileData);
|
||||
fprintf(stderr, "decompression error\n");
|
||||
|
@ -832,7 +830,7 @@ static int do_dec_benchmark(const char *pszInFilename, const char *pszOutFilenam
|
|||
}
|
||||
|
||||
long long nCurDecTime = t1 - t0;
|
||||
if (nBestDecTime == -1 || nBestDecTime > nCurDecTime)
|
||||
if (nBestDecTime == (size_t)-1 || nBestDecTime > nCurDecTime)
|
||||
nBestDecTime = nCurDecTime;
|
||||
}
|
||||
|
||||
|
@ -852,7 +850,7 @@ static int do_dec_benchmark(const char *pszInFilename, const char *pszOutFilenam
|
|||
free(pFileData);
|
||||
|
||||
fprintf(stdout, "format: LZSA%d\n", nFormatVersion);
|
||||
fprintf(stdout, "decompressed size: %zd bytes\n", nActualDecompressedSize);
|
||||
fprintf(stdout, "decompressed size: %zu bytes\n", nActualDecompressedSize);
|
||||
fprintf(stdout, "decompression time: %lld microseconds (%g Mb/s)\n", nBestDecTime, ((double)nActualDecompressedSize / 1024.0) / ((double)nBestDecTime / 1000.0));
|
||||
|
||||
return 0;
|
||||
|
@ -1037,11 +1035,11 @@ int main(int argc, char **argv) {
|
|||
nArgsError = 1;
|
||||
}
|
||||
else if (!strcmp(argv[i], "-stats")) {
|
||||
if ((nOptions & OPT_STATS) == 0) {
|
||||
nOptions |= OPT_STATS;
|
||||
}
|
||||
else
|
||||
nArgsError = 1;
|
||||
if ((nOptions & OPT_STATS) == 0) {
|
||||
nOptions |= OPT_STATS;
|
||||
}
|
||||
else
|
||||
nArgsError = 1;
|
||||
}
|
||||
else {
|
||||
if (!pszInFilename)
|
||||
|
|
|
@ -33,7 +33,6 @@
|
|||
#include <string.h>
|
||||
#include "matchfinder.h"
|
||||
#include "format.h"
|
||||
#include "lib.h"
|
||||
|
||||
/**
|
||||
* Hash index into TAG_BITS
|
||||
|
@ -77,7 +76,7 @@ int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *p
|
|||
PLCP[i] = 0;
|
||||
continue;
|
||||
}
|
||||
int nMaxLen = (i > Phi[i]) ? (nInWindowSize - i) : (nInWindowSize - Phi[i]);
|
||||
const int nMaxLen = (i > Phi[i]) ? (nInWindowSize - i) : (nInWindowSize - Phi[i]);
|
||||
while (nCurLen < nMaxLen && pInWindow[i + nCurLen] == pInWindow[Phi[i] + nCurLen]) nCurLen++;
|
||||
PLCP[i] = nCurLen;
|
||||
if (nCurLen > 0)
|
||||
|
@ -88,11 +87,11 @@ int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *p
|
|||
* saves us from having to build the inverse suffix array index, as the LCP is calculated without it using this method,
|
||||
* and the interval builder below doesn't need it either. */
|
||||
intervals[0] &= POS_MASK;
|
||||
int nMinMatchSize = pCompressor->min_match_size;
|
||||
const int nMinMatchSize = pCompressor->min_match_size;
|
||||
|
||||
if (pCompressor->format_version >= 2) {
|
||||
for (i = 1; i < nInWindowSize; i++) {
|
||||
int nIndex = (int)(intervals[i] & POS_MASK);
|
||||
const int nIndex = (int)(intervals[i] & POS_MASK);
|
||||
int nLen = PLCP[nIndex];
|
||||
if (nLen < nMinMatchSize)
|
||||
nLen = 0;
|
||||
|
@ -106,7 +105,7 @@ int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *p
|
|||
}
|
||||
else {
|
||||
for (i = 1; i < nInWindowSize; i++) {
|
||||
int nIndex = (int)(intervals[i] & POS_MASK);
|
||||
const int nIndex = (int)(intervals[i] & POS_MASK);
|
||||
int nLen = PLCP[nIndex];
|
||||
if (nLen < nMinMatchSize)
|
||||
nLen = 0;
|
||||
|
@ -196,14 +195,15 @@ int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *p
|
|||
*
|
||||
* @return number of matches
|
||||
*/
|
||||
int lzsa_find_matches_at(lzsa_compressor *pCompressor, const int nOffset, lzsa_match *pMatches, const int nMaxMatches, const int nInWindowSize) {
|
||||
static int lzsa_find_matches_at(lzsa_compressor *pCompressor, const int nOffset, lzsa_match *pMatches, const int nMaxMatches, const int nInWindowSize) {
|
||||
unsigned int *intervals = pCompressor->intervals;
|
||||
unsigned int *pos_data = pCompressor->pos_data;
|
||||
unsigned int ref;
|
||||
unsigned int super_ref;
|
||||
unsigned int match_pos;
|
||||
lzsa_match *matchptr;
|
||||
int nPrevOffset = 0;
|
||||
unsigned int nPrevOffset = 0;
|
||||
unsigned char nV1OffsetFound[2] = { 0, 0 };
|
||||
|
||||
/**
|
||||
* Find matches using intervals
|
||||
|
@ -240,11 +240,11 @@ int lzsa_find_matches_at(lzsa_compressor *pCompressor, const int nOffset, lzsa_m
|
|||
|
||||
if (pCompressor->format_version >= 2 && nInWindowSize < 65536) {
|
||||
if ((matchptr - pMatches) < nMaxMatches) {
|
||||
int nMatchOffset = (int)(nOffset - match_pos);
|
||||
const unsigned int nMatchOffset = (const unsigned int)(nOffset - match_pos);
|
||||
|
||||
if (nMatchOffset <= MAX_OFFSET) {
|
||||
matchptr->length = (unsigned short)(ref >> (LCP_SHIFT + TAG_BITS));
|
||||
matchptr->offset = (unsigned short)nMatchOffset;
|
||||
matchptr->length = (const unsigned short)(ref >> (LCP_SHIFT + TAG_BITS));
|
||||
matchptr->offset = (const unsigned short)nMatchOffset;
|
||||
matchptr++;
|
||||
|
||||
nPrevOffset = nMatchOffset;
|
||||
|
@ -258,11 +258,11 @@ int lzsa_find_matches_at(lzsa_compressor *pCompressor, const int nOffset, lzsa_m
|
|||
|
||||
if (pCompressor->format_version >= 2 && nInWindowSize < 65536) {
|
||||
if ((matchptr - pMatches) < nMaxMatches) {
|
||||
int nMatchOffset = (int)(nOffset - match_pos);
|
||||
const unsigned int nMatchOffset = (const unsigned int)(nOffset - match_pos);
|
||||
|
||||
if (nMatchOffset <= MAX_OFFSET && nMatchOffset != nPrevOffset) {
|
||||
matchptr->length = ((unsigned short)(ref >> (LCP_SHIFT + TAG_BITS))) | 0x8000;
|
||||
matchptr->offset = (unsigned short)nMatchOffset;
|
||||
if (nMatchOffset <= MAX_OFFSET) {
|
||||
matchptr->length = ((const unsigned short)(ref >> (LCP_SHIFT + TAG_BITS))) | 0x8000;
|
||||
matchptr->offset = (const unsigned short)nMatchOffset;
|
||||
matchptr++;
|
||||
|
||||
nPrevOffset = nMatchOffset;
|
||||
|
@ -277,17 +277,30 @@ int lzsa_find_matches_at(lzsa_compressor *pCompressor, const int nOffset, lzsa_m
|
|||
pos_data[match_pos] = ref;
|
||||
|
||||
if ((matchptr - pMatches) < nMaxMatches) {
|
||||
int nMatchOffset = (int)(nOffset - match_pos);
|
||||
const unsigned int nMatchOffset = (const unsigned int)(nOffset - match_pos);
|
||||
|
||||
if (nMatchOffset <= MAX_OFFSET && nMatchOffset != nPrevOffset) {
|
||||
if (pCompressor->format_version >= 2) {
|
||||
matchptr->length = (unsigned short)(ref >> (LCP_SHIFT + TAG_BITS));
|
||||
matchptr->length = (const unsigned short)(ref >> (LCP_SHIFT + TAG_BITS));
|
||||
matchptr->offset = (const unsigned short)nMatchOffset;
|
||||
matchptr++;
|
||||
|
||||
nPrevOffset = nMatchOffset;
|
||||
}
|
||||
else {
|
||||
matchptr->length = (unsigned short)(ref >> LCP_SHIFT);
|
||||
unsigned int nV1OffsetType = (nMatchOffset <= 256) ? 0 : 1;
|
||||
|
||||
if (!nV1OffsetFound[nV1OffsetType]) {
|
||||
matchptr->length = (const unsigned short)(ref >> LCP_SHIFT);
|
||||
matchptr->offset = (const unsigned short)nMatchOffset;
|
||||
|
||||
if (matchptr->length < 256)
|
||||
nV1OffsetFound[nV1OffsetType] = 1;
|
||||
matchptr++;
|
||||
|
||||
nPrevOffset = nMatchOffset;
|
||||
}
|
||||
}
|
||||
matchptr->offset = (unsigned short)nMatchOffset;
|
||||
matchptr++;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -298,13 +311,14 @@ int lzsa_find_matches_at(lzsa_compressor *pCompressor, const int nOffset, lzsa_m
|
|||
|
||||
if (pCompressor->format_version >= 2 && nInWindowSize < 65536) {
|
||||
if ((matchptr - pMatches) < nMaxMatches) {
|
||||
int nMatchOffset = (int)(nOffset - match_pos);
|
||||
const unsigned int nMatchOffset = (const unsigned int)(nOffset - match_pos);
|
||||
|
||||
if (nMatchOffset <= MAX_OFFSET && nMatchOffset != nPrevOffset) {
|
||||
matchptr->length = ((unsigned short)(ref >> (LCP_SHIFT + TAG_BITS))) | 0x8000;
|
||||
matchptr->offset = (unsigned short)nMatchOffset;
|
||||
if (nMatchOffset <= MAX_OFFSET) {
|
||||
const unsigned short nMatchLen = ((const unsigned short)(ref >> (LCP_SHIFT + TAG_BITS)));
|
||||
|
||||
if ((matchptr->length & 0x7fff) > 2) {
|
||||
if (nMatchLen > 2) {
|
||||
matchptr->length = nMatchLen | 0x8000;
|
||||
matchptr->offset = (const unsigned short)nMatchOffset;
|
||||
matchptr++;
|
||||
|
||||
nPrevOffset = nMatchOffset;
|
||||
|
@ -348,12 +362,10 @@ void lzsa_find_all_matches(lzsa_compressor *pCompressor, const int nMatchesPerOf
|
|||
int i;
|
||||
|
||||
for (i = nStartOffset; i < nEndOffset; i++) {
|
||||
int nMatches = lzsa_find_matches_at(pCompressor, i, pMatch, nMatchesPerOffset, nEndOffset - nStartOffset);
|
||||
const int nMatches = lzsa_find_matches_at(pCompressor, i, pMatch, nMatchesPerOffset, nEndOffset - nStartOffset);
|
||||
|
||||
while (nMatches < nMatchesPerOffset) {
|
||||
pMatch[nMatches].length = 0;
|
||||
pMatch[nMatches].offset = 0;
|
||||
nMatches++;
|
||||
if (nMatches < nMatchesPerOffset) {
|
||||
memset(pMatch + nMatches, 0, (nMatchesPerOffset - nMatches) * sizeof(lzsa_match));
|
||||
}
|
||||
|
||||
pMatch += nMatchesPerOffset;
|
||||
|
|
|
@ -33,14 +33,12 @@
|
|||
#ifndef _MATCHFINDER_H
|
||||
#define _MATCHFINDER_H
|
||||
|
||||
#include "shrink_context.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Forward declarations */
|
||||
typedef struct _lzsa_match lzsa_match;
|
||||
typedef struct _lzsa_compressor lzsa_compressor;
|
||||
|
||||
/**
|
||||
* Parse input data, build suffix array and overlaid data structures to speed up match finding
|
||||
*
|
||||
|
@ -52,19 +50,6 @@ typedef struct _lzsa_compressor lzsa_compressor;
|
|||
*/
|
||||
int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nInWindowSize);
|
||||
|
||||
/**
|
||||
* Find matches at the specified offset in the input window
|
||||
*
|
||||
* @param pCompressor compression context
|
||||
* @param nOffset offset to find matches at, in the input window
|
||||
* @param pMatches pointer to returned matches
|
||||
* @param nMaxMatches maximum number of matches to return (0 for none)
|
||||
* @param nInWindowSize total input size in bytes (previously compressed bytes + bytes to compress)
|
||||
*
|
||||
* @return number of matches
|
||||
*/
|
||||
int lzsa_find_matches_at(lzsa_compressor *pCompressor, const int nOffset, lzsa_match *pMatches, const int nMaxMatches, const int nInWindowSize);
|
||||
|
||||
/**
|
||||
* Skip previously compressed bytes
|
||||
*
|
||||
|
|
|
@ -67,7 +67,7 @@ static inline int lzsa_get_literals_varlen_size_v1(const int nLength) {
|
|||
* @param nOutOffset current write index into output buffer
|
||||
* @param nLength literals length
|
||||
*/
|
||||
static inline int lzsa_write_literals_varlen_v1(unsigned char *pOutData, int nOutOffset, int nLength) {
|
||||
static inline int lzsa_write_literals_varlen_v1(unsigned char *pOutData, int nOutOffset, const int nLength) {
|
||||
if (nLength >= LITERALS_RUN_LEN_V1) {
|
||||
if (nLength < 256)
|
||||
pOutData[nOutOffset++] = nLength - LITERALS_RUN_LEN_V1;
|
||||
|
@ -118,7 +118,7 @@ static inline int lzsa_get_match_varlen_size_v1(const int nLength) {
|
|||
* @param nOutOffset current write index into output buffer
|
||||
* @param nLength encoded match length (actual match length - MIN_MATCH_SIZE_V1)
|
||||
*/
|
||||
static inline int lzsa_write_match_varlen_v1(unsigned char *pOutData, int nOutOffset, int nLength) {
|
||||
static inline int lzsa_write_match_varlen_v1(unsigned char *pOutData, int nOutOffset, const int nLength) {
|
||||
if (nLength >= MATCH_RUN_LEN_V1) {
|
||||
if ((nLength + MIN_MATCH_SIZE_V1) < 256)
|
||||
pOutData[nOutOffset++] = nLength - MATCH_RUN_LEN_V1;
|
||||
|
@ -141,7 +141,7 @@ static inline int lzsa_write_match_varlen_v1(unsigned char *pOutData, int nOutOf
|
|||
/**
|
||||
* Get offset encoding cost in bits
|
||||
*
|
||||
* @param nMatchOffset offset to get cost of
|
||||
* @param __nMatchOffset offset to get cost of
|
||||
*
|
||||
* @return cost in bits
|
||||
*/
|
||||
|
@ -153,9 +153,10 @@ static inline int lzsa_write_match_varlen_v1(unsigned char *pOutData, int nOutOf
|
|||
* @param pCompressor compression context
|
||||
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
|
||||
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
|
||||
* @param nReduce non-zero to reduce the number of tokens when the path costs are equal, zero not to
|
||||
*/
|
||||
static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset, const int nReduce) {
|
||||
lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << ARRIVALS_PER_POSITION_SHIFT);
|
||||
static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset, const int nReduce) {
|
||||
lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << ARRIVALS_PER_POSITION_SHIFT_V1);
|
||||
const int nMinMatchSize = pCompressor->min_match_size;
|
||||
const int nFavorRatio = (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) ? 1 : 0;
|
||||
const int nModeSwitchPenalty = nFavorRatio ? 0 : MODESWITCH_PENALTY;
|
||||
|
@ -164,21 +165,22 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
|
|||
|
||||
if ((nEndOffset - nStartOffset) > BLOCK_SIZE) return;
|
||||
|
||||
for (i = (nStartOffset << ARRIVALS_PER_POSITION_SHIFT); i != ((nEndOffset + 1) << ARRIVALS_PER_POSITION_SHIFT); i += NARRIVALS_PER_POSITION_V2_MAX) {
|
||||
for (i = (nStartOffset << ARRIVALS_PER_POSITION_SHIFT_V1); i != ((nEndOffset + 1) << ARRIVALS_PER_POSITION_SHIFT_V1); i += NARRIVALS_PER_POSITION_V1) {
|
||||
lzsa_arrival* cur_arrival = &arrival[i];
|
||||
int j;
|
||||
|
||||
memset(arrival + i, 0, sizeof(lzsa_arrival) * NARRIVALS_PER_POSITION_V2_MAX);
|
||||
memset(cur_arrival, 0, sizeof(lzsa_arrival) * NARRIVALS_PER_POSITION_V1);
|
||||
|
||||
for (j = 0; j < NARRIVALS_PER_POSITION_V2_MAX; j++)
|
||||
arrival[i + j].cost = 0x40000000;
|
||||
for (j = 0; j < NARRIVALS_PER_POSITION_V1; j++)
|
||||
cur_arrival[j].cost = 0x40000000;
|
||||
}
|
||||
|
||||
arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT].cost = 0;
|
||||
arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT].from_slot = -1;
|
||||
arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT_V1].cost = 0;
|
||||
arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT_V1].from_slot = -1;
|
||||
|
||||
for (i = nStartOffset; i != nEndOffset; i++) {
|
||||
lzsa_arrival* cur_arrival = &arrival[i << ARRIVALS_PER_POSITION_SHIFT];
|
||||
lzsa_arrival* pDestLiteralSlots = &cur_arrival[1 << ARRIVALS_PER_POSITION_SHIFT];
|
||||
lzsa_arrival* cur_arrival = &arrival[i << ARRIVALS_PER_POSITION_SHIFT_V1];
|
||||
lzsa_arrival* pDestLiteralSlots = &cur_arrival[1 << ARRIVALS_PER_POSITION_SHIFT_V1];
|
||||
int j, m;
|
||||
|
||||
for (j = 0; j < NARRIVALS_PER_POSITION_V1 && cur_arrival[j].from_slot; j++) {
|
||||
|
@ -205,7 +207,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
|
|||
pDestArrival->cost = nCodingChoiceCost;
|
||||
pDestArrival->rep_offset = cur_arrival[j].rep_offset;
|
||||
pDestArrival->from_slot = j + 1;
|
||||
pDestArrival->from_pos = i;
|
||||
pDestArrival->from_pos = i - nStartOffset;
|
||||
pDestArrival->match_len = 0;
|
||||
pDestArrival->num_literals = nNumLiterals;
|
||||
pDestArrival->score = nScore;
|
||||
|
@ -217,28 +219,27 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
|
|||
const lzsa_match *match = pCompressor->match + ((i - nStartOffset) << MATCHES_PER_INDEX_SHIFT_V1);
|
||||
const int nNumArrivalsForThisPos = j;
|
||||
|
||||
for (m = 0; m < NMATCHES_PER_INDEX_V1 && match[m].length; m++) {
|
||||
int nMatchLen = match[m].length;
|
||||
const int nMatchOffsetCost = lzsa_get_offset_cost_v1(match[m].offset);
|
||||
int nStartingMatchLen, k;
|
||||
if (nNumArrivalsForThisPos != 0) {
|
||||
for (m = 0; m < NMATCHES_PER_INDEX_V1 && match[m].length; m++) {
|
||||
int nMatchLen = match[m].length;
|
||||
const int nMatchOffsetCost = lzsa_get_offset_cost_v1(match[m].offset);
|
||||
int nStartingMatchLen, k;
|
||||
|
||||
if ((i + nMatchLen) > nEndOffset)
|
||||
nMatchLen = nEndOffset - i;
|
||||
if ((i + nMatchLen) > nEndOffset)
|
||||
nMatchLen = nEndOffset - i;
|
||||
|
||||
if (nMatchLen >= LEAVE_ALONE_MATCH_SIZE)
|
||||
nStartingMatchLen = nMatchLen;
|
||||
else
|
||||
nStartingMatchLen = nMinMatchSize;
|
||||
for (k = nStartingMatchLen; k <= nMatchLen; k++) {
|
||||
const int nMatchLenCost = lzsa_get_match_varlen_size_v1(k - MIN_MATCH_SIZE_V1);
|
||||
lzsa_arrival *pDestSlots = &arrival[(i + k) << ARRIVALS_PER_POSITION_SHIFT];
|
||||
if (nMatchLen >= LEAVE_ALONE_MATCH_SIZE)
|
||||
nStartingMatchLen = nMatchLen;
|
||||
else
|
||||
nStartingMatchLen = nMinMatchSize;
|
||||
for (k = nStartingMatchLen; k <= nMatchLen; k++) {
|
||||
const int nMatchLenCost = lzsa_get_match_varlen_size_v1(k - MIN_MATCH_SIZE_V1);
|
||||
|
||||
for (j = 0; j < nNumArrivalsForThisPos; j++) {
|
||||
const int nPrevCost = cur_arrival[j].cost;
|
||||
int nCodingChoiceCost = nPrevCost + 8 /* token */ /* the actual cost of the literals themselves accumulates up the chain */ + nMatchOffsetCost + nMatchLenCost;
|
||||
lzsa_arrival* pDestSlots = &cur_arrival[k << ARRIVALS_PER_POSITION_SHIFT_V1];
|
||||
int nCodingChoiceCost = cur_arrival[0].cost + 8 /* token */ /* the actual cost of the literals themselves accumulates up the chain */ + nMatchOffsetCost + nMatchLenCost;
|
||||
int exists = 0, n;
|
||||
|
||||
if (!cur_arrival[j].num_literals)
|
||||
if (!cur_arrival[0].num_literals)
|
||||
nCodingChoiceCost += nModeSwitchPenalty;
|
||||
|
||||
for (n = 0;
|
||||
|
@ -251,51 +252,35 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
|
|||
}
|
||||
|
||||
if (!exists) {
|
||||
const int nScore = cur_arrival[j].score + 5;
|
||||
int nNonRepMatchIdx = -1;
|
||||
const int nScore = cur_arrival[0].score + 5;
|
||||
|
||||
for (n = 0; n < NARRIVALS_PER_POSITION_V1 /* we only need the literals + short match cost + long match cost cases */; n++) {
|
||||
if (nCodingChoiceCost < pDestSlots[n].cost ||
|
||||
(nCodingChoiceCost == pDestSlots[n].cost && nScore < (pDestSlots[n].score + nDisableScore))) {
|
||||
nNonRepMatchIdx = n;
|
||||
break;
|
||||
}
|
||||
if (nCodingChoiceCost < pDestSlots[0].cost ||
|
||||
(nCodingChoiceCost == pDestSlots[0].cost && nScore < (pDestSlots[0].score + nDisableScore))) {
|
||||
memmove(&pDestSlots[1],
|
||||
&pDestSlots[0],
|
||||
sizeof(lzsa_arrival) * (NARRIVALS_PER_POSITION_V1 - 1));
|
||||
|
||||
pDestSlots->cost = nCodingChoiceCost;
|
||||
pDestSlots->rep_offset = match[m].offset;
|
||||
pDestSlots->from_slot = 1;
|
||||
pDestSlots->from_pos = i - nStartOffset;
|
||||
pDestSlots->match_len = k;
|
||||
pDestSlots->num_literals = 0;
|
||||
pDestSlots->score = nScore;
|
||||
}
|
||||
|
||||
if (nNonRepMatchIdx >= 0) {
|
||||
memmove(&pDestSlots[nNonRepMatchIdx + 1],
|
||||
&pDestSlots[nNonRepMatchIdx],
|
||||
sizeof(lzsa_arrival) * (NARRIVALS_PER_POSITION_V1 - nNonRepMatchIdx - 1));
|
||||
|
||||
lzsa_arrival* pDestArrival = &pDestSlots[nNonRepMatchIdx];
|
||||
pDestArrival->cost = nCodingChoiceCost;
|
||||
pDestArrival->rep_offset = match[m].offset;
|
||||
pDestArrival->from_slot = j + 1;
|
||||
pDestArrival->from_pos = i;
|
||||
pDestArrival->match_len = k;
|
||||
pDestArrival->num_literals = 0;
|
||||
pDestArrival->score = nScore;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const lzsa_arrival *end_arrival = &arrival[(i << ARRIVALS_PER_POSITION_SHIFT) + 0];
|
||||
const lzsa_arrival *end_arrival = &arrival[i << ARRIVALS_PER_POSITION_SHIFT_V1];
|
||||
lzsa_match *pBestMatch = pCompressor->best_match - nStartOffset;
|
||||
|
||||
while (end_arrival->from_slot > 0 && end_arrival->from_pos >= 0 && end_arrival->from_pos < nEndOffset) {
|
||||
pBestMatch[end_arrival->from_pos].length = end_arrival->match_len;
|
||||
if (end_arrival->match_len)
|
||||
pBestMatch[end_arrival->from_pos].offset = end_arrival->rep_offset;
|
||||
else
|
||||
pBestMatch[end_arrival->from_pos].offset = 0;
|
||||
|
||||
end_arrival = &arrival[(end_arrival->from_pos << ARRIVALS_PER_POSITION_SHIFT) + (end_arrival->from_slot - 1)];
|
||||
while (end_arrival->from_slot > 0 && (end_arrival->from_pos + nStartOffset) < nEndOffset) {
|
||||
pBestMatch[end_arrival->from_pos + nStartOffset].length = end_arrival->match_len;
|
||||
pBestMatch[end_arrival->from_pos + nStartOffset].offset = (end_arrival->match_len) ? end_arrival->rep_offset: 0;
|
||||
end_arrival = &arrival[((end_arrival->from_pos + nStartOffset) << ARRIVALS_PER_POSITION_SHIFT_V1) + (end_arrival->from_slot - 1)];
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -305,13 +290,13 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
|
|||
*
|
||||
* @param pCompressor compression context
|
||||
* @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress)
|
||||
* @param pBestMatch optimal matches to emit
|
||||
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
|
||||
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
|
||||
*
|
||||
* @return non-zero if the number of tokens was reduced, 0 if it wasn't
|
||||
*/
|
||||
static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) {
|
||||
static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset) {
|
||||
lzsa_match *pBestMatch = pCompressor->best_match - nStartOffset;
|
||||
int i;
|
||||
int nNumLiterals = 0;
|
||||
int nDidReduce = 0;
|
||||
|
@ -381,7 +366,7 @@ static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const un
|
|||
pBestMatch[i + pMatch->length].length)) {
|
||||
|
||||
int nCurPartialSize = lzsa_get_match_varlen_size_v1(pMatch->length - MIN_MATCH_SIZE_V1);
|
||||
nCurPartialSize += 8 /* token */ + lzsa_get_literals_varlen_size_v1(0) + ((pBestMatch[i + pMatch->length].offset <= 256) ? 8 : 16) /* match offset */ + lzsa_get_match_varlen_size_v1(pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V1);
|
||||
nCurPartialSize += 8 /* token */ + /* lzsa_get_literals_varlen_size_v1(0) + */ ((pBestMatch[i + pMatch->length].offset <= 256) ? 8 : 16) /* match offset */ + lzsa_get_match_varlen_size_v1(pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V1);
|
||||
|
||||
const int nReducedPartialSize = lzsa_get_match_varlen_size_v1(pMatch->length + pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V1);
|
||||
|
||||
|
@ -410,60 +395,10 @@ static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const un
|
|||
return nDidReduce;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get compressed data block size
|
||||
*
|
||||
* @param pCompressor compression context
|
||||
* @param pBestMatch optimal matches to emit
|
||||
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
|
||||
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
|
||||
*
|
||||
* @return size of compressed data that will be written to output buffer
|
||||
*/
|
||||
static int lzsa_get_compressed_size_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) {
|
||||
int i;
|
||||
int nNumLiterals = 0;
|
||||
int nCompressedSize = 0;
|
||||
|
||||
for (i = nStartOffset; i < nEndOffset; ) {
|
||||
const lzsa_match *pMatch = pBestMatch + i;
|
||||
|
||||
if (pMatch->length >= MIN_MATCH_SIZE_V1) {
|
||||
const int nMatchOffset = pMatch->offset;
|
||||
const int nMatchLen = pMatch->length;
|
||||
const int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE_V1;
|
||||
const int nTokenLongOffset = (nMatchOffset <= 256) ? 0x00 : 0x80;
|
||||
const int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3) + (nTokenLongOffset ? 16 : 8) /* match offset */ + lzsa_get_match_varlen_size_v1(nEncodedMatchLen);
|
||||
|
||||
nCompressedSize += nCommandSize;
|
||||
nNumLiterals = 0;
|
||||
i += nMatchLen;
|
||||
}
|
||||
else {
|
||||
nNumLiterals++;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3);
|
||||
|
||||
nCompressedSize += nCommandSize;
|
||||
nNumLiterals = 0;
|
||||
}
|
||||
|
||||
if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
|
||||
nCompressedSize += 8 * 4;
|
||||
}
|
||||
|
||||
return nCompressedSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit block of compressed data
|
||||
*
|
||||
* @param pCompressor compression context
|
||||
* @param pBestMatch optimal matches to emit
|
||||
* @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress)
|
||||
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
|
||||
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
|
||||
|
@ -472,7 +407,8 @@ static int lzsa_get_compressed_size_v1(lzsa_compressor *pCompressor, lzsa_match
|
|||
*
|
||||
* @return size of compressed data in output buffer, or -1 if the data is uncompressible
|
||||
*/
|
||||
static int lzsa_write_block_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) {
|
||||
static int lzsa_write_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) {
|
||||
const lzsa_match *pBestMatch = pCompressor->best_match - nStartOffset;
|
||||
int i;
|
||||
int nNumLiterals = 0;
|
||||
int nInFirstLiteralOffset = 0;
|
||||
|
@ -550,7 +486,7 @@ static int lzsa_write_block_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMa
|
|||
i += nMatchLen;
|
||||
|
||||
if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
|
||||
int nCurSafeDist = (i - nStartOffset) - nOutOffset;
|
||||
const int nCurSafeDist = (i - nStartOffset) - nOutOffset;
|
||||
if (nCurSafeDist >= 0 && pCompressor->safe_dist < nCurSafeDist)
|
||||
pCompressor->safe_dist = nCurSafeDist;
|
||||
}
|
||||
|
@ -575,7 +511,7 @@ static int lzsa_write_block_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMa
|
|||
if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK)
|
||||
pOutData[nOutOffset++] = (nTokenLiteralsLen << 4) | 0x0f;
|
||||
else
|
||||
pOutData[nOutOffset++] = (nTokenLiteralsLen << 4) | 0x00;
|
||||
pOutData[nOutOffset++] = (nTokenLiteralsLen << 4) /* | 0x00 */;
|
||||
nOutOffset = lzsa_write_literals_varlen_v1(pOutData, nOutOffset, nNumLiterals);
|
||||
|
||||
if (nNumLiterals < pCompressor->stats.min_literals || pCompressor->stats.min_literals == -1)
|
||||
|
@ -588,11 +524,10 @@ static int lzsa_write_block_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMa
|
|||
if (nNumLiterals != 0) {
|
||||
memcpy(pOutData + nOutOffset, pInWindow + nInFirstLiteralOffset, nNumLiterals);
|
||||
nOutOffset += nNumLiterals;
|
||||
nNumLiterals = 0;
|
||||
}
|
||||
|
||||
if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
|
||||
int nCurSafeDist = (i - nStartOffset) - nOutOffset;
|
||||
const int nCurSafeDist = (i - nStartOffset) - nOutOffset;
|
||||
if (nCurSafeDist >= 0 && pCompressor->safe_dist < nCurSafeDist)
|
||||
pCompressor->safe_dist = nCurSafeDist;
|
||||
}
|
||||
|
@ -628,11 +563,11 @@ static int lzsa_write_block_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMa
|
|||
* @return size of compressed data in output buffer, or -1 if the data is uncompressible
|
||||
*/
|
||||
static int lzsa_write_raw_uncompressed_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) {
|
||||
int nNumLiterals = nEndOffset - nStartOffset;
|
||||
const int nNumLiterals = nEndOffset - nStartOffset;
|
||||
const int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V1) ? LITERALS_RUN_LEN_V1 : nNumLiterals;
|
||||
int nOutOffset = 0;
|
||||
|
||||
int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3) + 4;
|
||||
const int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3) + 4;
|
||||
if ((nOutOffset + (nCommandSize >> 3)) > nMaxOutDataSize)
|
||||
return -1;
|
||||
|
||||
|
@ -644,7 +579,6 @@ static int lzsa_write_raw_uncompressed_block_v1(lzsa_compressor *pCompressor, co
|
|||
if (nNumLiterals != 0) {
|
||||
memcpy(pOutData + nOutOffset, pInWindow + nStartOffset, nNumLiterals);
|
||||
nOutOffset += nNumLiterals;
|
||||
nNumLiterals = 0;
|
||||
}
|
||||
|
||||
pCompressor->num_commands++;
|
||||
|
@ -672,45 +606,28 @@ static int lzsa_write_raw_uncompressed_block_v1(lzsa_compressor *pCompressor, co
|
|||
* @return size of compressed data in output buffer, or -1 if the data is uncompressible
|
||||
*/
|
||||
int lzsa_optimize_and_write_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) {
|
||||
int nResult, nBaseCompressedSize;
|
||||
int nResult;
|
||||
|
||||
/* Compress optimally without breaking ties in favor of less tokens */
|
||||
|
||||
memset(pCompressor->best_match, 0, BLOCK_SIZE * sizeof(lzsa_match));
|
||||
lzsa_optimize_forward_v1(pCompressor, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 0 /* reduce */);
|
||||
|
||||
if (nInDataSize < 65536) {
|
||||
lzsa_optimize_forward_v1(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 1 /* reduce */);
|
||||
}
|
||||
else {
|
||||
lzsa_optimize_forward_v1(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 0 /* reduce */);
|
||||
}
|
||||
|
||||
int nDidReduce;
|
||||
int nPasses = 0;
|
||||
do {
|
||||
nDidReduce = lzsa_optimize_command_count_v1(pCompressor, pInWindow, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
|
||||
nDidReduce = lzsa_optimize_command_count_v1(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
|
||||
nPasses++;
|
||||
} while (nDidReduce && nPasses < 20);
|
||||
|
||||
nBaseCompressedSize = lzsa_get_compressed_size_v1(pCompressor, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
|
||||
lzsa_match *pBestMatch = pCompressor->best_match - nPreviousBlockSize;
|
||||
|
||||
if (nBaseCompressedSize > 0 && nInDataSize < 65536) {
|
||||
int nReducedCompressedSize;
|
||||
|
||||
/* Compress optimally and do break ties in favor of less tokens */
|
||||
memset(pCompressor->improved_match, 0, BLOCK_SIZE * sizeof(lzsa_match));
|
||||
lzsa_optimize_forward_v1(pCompressor, pCompressor->improved_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 1 /* reduce */);
|
||||
|
||||
nPasses = 0;
|
||||
do {
|
||||
nDidReduce = lzsa_optimize_command_count_v1(pCompressor, pInWindow, pCompressor->improved_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
|
||||
nPasses++;
|
||||
} while (nDidReduce && nPasses < 20);
|
||||
|
||||
nReducedCompressedSize = lzsa_get_compressed_size_v1(pCompressor, pCompressor->improved_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
|
||||
if (nReducedCompressedSize > 0 && nReducedCompressedSize <= nBaseCompressedSize) {
|
||||
/* Pick the parse with the reduced number of tokens as it didn't negatively affect the size */
|
||||
pBestMatch = pCompressor->improved_match - nPreviousBlockSize;
|
||||
}
|
||||
}
|
||||
|
||||
nResult = lzsa_write_block_v1(pCompressor, pBestMatch, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
|
||||
if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
|
||||
nResult = lzsa_write_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
|
||||
if (nResult < 0 && (pCompressor->flags & LZSA_FLAG_RAW_BLOCK)) {
|
||||
nResult = lzsa_write_raw_uncompressed_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
|
||||
}
|
||||
|
||||
|
|
|
@ -33,8 +33,11 @@
|
|||
#ifndef _SHRINK_BLOCK_V1_H
|
||||
#define _SHRINK_BLOCK_V1_H
|
||||
|
||||
/* Forward declarations */
|
||||
typedef struct _lzsa_compressor lzsa_compressor;
|
||||
#include "shrink_context.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Select the most optimal matches, reduce the token count if possible, and then emit a block of compressed LZSA1 data
|
||||
|
@ -50,4 +53,8 @@ typedef struct _lzsa_compressor lzsa_compressor;
|
|||
*/
|
||||
int lzsa_optimize_and_write_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _SHRINK_BLOCK_V1_H */
|
||||
|
|
|
@ -80,9 +80,9 @@ static inline int lzsa_get_literals_varlen_size_v2(const int nLength) {
|
|||
}
|
||||
else {
|
||||
if (nLength < 256)
|
||||
return 4+8;
|
||||
return 4 + 8;
|
||||
else {
|
||||
return 4+24;
|
||||
return 4 + 24;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -138,7 +138,7 @@ static inline int lzsa_get_match_varlen_size_v2(const int nLength) {
|
|||
return 4;
|
||||
else {
|
||||
if ((nLength + MIN_MATCH_SIZE_V2) < 256)
|
||||
return 4+8;
|
||||
return 4 + 8;
|
||||
else {
|
||||
return 4 + 24;
|
||||
}
|
||||
|
@ -192,8 +192,8 @@ static inline int lzsa_write_match_varlen_v2(unsigned char *pOutData, int nOutOf
|
|||
* @param nDepth current insertion depth
|
||||
*/
|
||||
static void lzsa_insert_forward_match_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int i, const int nMatchOffset, const int nStartOffset, const int nEndOffset, const int nDepth) {
|
||||
const lzsa_arrival *arrival = pCompressor->arrival + ((i - nStartOffset) << ARRIVALS_PER_POSITION_SHIFT);
|
||||
const int *rle_len = (int*)pCompressor->intervals /* reuse */;
|
||||
const lzsa_arrival *arrival = pCompressor->arrival + ((i - nStartOffset) << ARRIVALS_PER_POSITION_SHIFT_V2);
|
||||
const int *rle_len = (const int*)pCompressor->intervals /* reuse */;
|
||||
lzsa_match* visited = ((lzsa_match*)pCompressor->pos_data) - nStartOffset /* reuse */;
|
||||
int j;
|
||||
|
||||
|
@ -223,17 +223,17 @@ static void lzsa_insert_forward_match_v2(lzsa_compressor *pCompressor, const uns
|
|||
const int nMinLen = (nLen0 < nLen1) ? nLen0 : nLen1;
|
||||
|
||||
if (nMinLen >= nRepLen || !memcmp(pInWindowStart + nMinLen, pInWindowStart + nMinLen - nMatchOffset, nRepLen - nMinLen)) {
|
||||
if (nRepLen >= MIN_MATCH_SIZE_V2) {
|
||||
if (nRepOffset) {
|
||||
int r;
|
||||
int r;
|
||||
|
||||
for (r = 0; fwd_match[r].length; r++) {
|
||||
if (fwd_match[r].offset == nMatchOffset) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
for (r = 0; fwd_match[r].length; r++) {
|
||||
if (fwd_match[r].offset == nMatchOffset) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (fwd_match[r].length == 0) {
|
||||
if (fwd_match[r].length == 0) {
|
||||
if (nRepLen >= MIN_MATCH_SIZE_V2) {
|
||||
if (nRepOffset) {
|
||||
int nMaxRepLen = nEndOffset - nRepPos;
|
||||
if (nMaxRepLen > LCP_MAX)
|
||||
nMaxRepLen = LCP_MAX;
|
||||
|
@ -252,7 +252,7 @@ static void lzsa_insert_forward_match_v2(lzsa_compressor *pCompressor, const uns
|
|||
while (pInWindowAtRepPos < pInWindowMax && pInWindowAtRepPos[0] == pInWindowAtRepPos[-nMatchOffset])
|
||||
pInWindowAtRepPos++;
|
||||
|
||||
fwd_match[r].length = (const unsigned int)(pInWindowAtRepPos - pInWindowStart);
|
||||
fwd_match[r].length = (const unsigned short)(pInWindowAtRepPos - pInWindowStart);
|
||||
fwd_match[r].offset = nMatchOffset;
|
||||
|
||||
if (nDepth < 9)
|
||||
|
@ -278,19 +278,18 @@ static void lzsa_insert_forward_match_v2(lzsa_compressor *pCompressor, const uns
|
|||
*
|
||||
* @param pCompressor compression context
|
||||
* @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress)
|
||||
* @param pBestMatch pointer to buffer for outputting optimal matches
|
||||
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
|
||||
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
|
||||
* @param nReduce non-zero to reduce the number of tokens when the path costs are equal, zero not to
|
||||
* @param nInsertForwardReps non-zero to insert forward repmatch candidates, zero to use the previously inserted candidates
|
||||
* @param nArrivalsPerPosition number of arrivals to record per input buffer position
|
||||
*/
|
||||
static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset, const int nReduce, const int nInsertForwardReps, const int nArrivalsPerPosition) {
|
||||
lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << ARRIVALS_PER_POSITION_SHIFT);
|
||||
static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, const int nReduce, const int nInsertForwardReps, const int nArrivalsPerPosition) {
|
||||
lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << ARRIVALS_PER_POSITION_SHIFT_V2);
|
||||
const int *rle_len = (const int*)pCompressor->intervals /* reuse */;
|
||||
lzsa_match *visited = ((lzsa_match*)pCompressor->pos_data) - nStartOffset /* reuse */;
|
||||
char *nRepSlotHandledMask = pCompressor->rep_slot_handled_mask;
|
||||
char *nRepLenHandledMask = pCompressor->rep_len_handled_mask;
|
||||
unsigned char *nRepSlotHandledMask = pCompressor->rep_slot_handled_mask;
|
||||
unsigned char *nRepLenHandledMask = pCompressor->rep_len_handled_mask;
|
||||
const int nModeSwitchPenalty = (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) ? 0 : MODESWITCH_PENALTY;
|
||||
const int nMinMatchSize = pCompressor->min_match_size;
|
||||
const int nDisableScore = nReduce ? 0 : (2 * BLOCK_SIZE);
|
||||
|
@ -300,24 +299,25 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
|
||||
if ((nEndOffset - nStartOffset) > BLOCK_SIZE) return;
|
||||
|
||||
for (i = (nStartOffset << ARRIVALS_PER_POSITION_SHIFT); i != ((nEndOffset + 1) << ARRIVALS_PER_POSITION_SHIFT); i += NARRIVALS_PER_POSITION_V2_MAX) {
|
||||
for (i = (nStartOffset << ARRIVALS_PER_POSITION_SHIFT_V2); i != ((nEndOffset + 1) << ARRIVALS_PER_POSITION_SHIFT_V2); i += NARRIVALS_PER_POSITION_V2_MAX) {
|
||||
lzsa_arrival *cur_arrival = &arrival[i];
|
||||
int j;
|
||||
|
||||
memset(arrival + i, 0, sizeof(lzsa_arrival) * NARRIVALS_PER_POSITION_V2_MAX);
|
||||
memset(cur_arrival, 0, sizeof(lzsa_arrival) * NARRIVALS_PER_POSITION_V2_MAX);
|
||||
|
||||
for (j = 0; j < NARRIVALS_PER_POSITION_V2_MAX; j++)
|
||||
arrival[i + j].cost = 0x40000000;
|
||||
cur_arrival[j].cost = 0x40000000;
|
||||
}
|
||||
|
||||
arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT].cost = 0;
|
||||
arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT].from_slot = -1;
|
||||
arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT_V2].cost = 0;
|
||||
arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT_V2].from_slot = -1;
|
||||
|
||||
if (nInsertForwardReps) {
|
||||
memset(visited + nStartOffset, 0, (nEndOffset - nStartOffset) * sizeof(lzsa_match));
|
||||
}
|
||||
|
||||
for (i = nStartOffset; i != nEndOffset; i++) {
|
||||
lzsa_arrival *cur_arrival = &arrival[i << ARRIVALS_PER_POSITION_SHIFT];
|
||||
lzsa_arrival *cur_arrival = &arrival[i << ARRIVALS_PER_POSITION_SHIFT_V2];
|
||||
lzsa_arrival *pDestLiteralSlots = &cur_arrival[NARRIVALS_PER_POSITION_V2_MAX];
|
||||
int j, m;
|
||||
|
||||
|
@ -328,21 +328,29 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
const int nNumLiterals = cur_arrival[j].num_literals + 1;
|
||||
const int nRepOffset = cur_arrival[j].rep_offset;
|
||||
|
||||
if (nNumLiterals == 1) {
|
||||
switch (nNumLiterals) {
|
||||
case 1:
|
||||
nCodingChoiceCost += nModeSwitchPenalty;
|
||||
}
|
||||
else if (nNumLiterals == LITERALS_RUN_LEN_V2) {
|
||||
break;
|
||||
|
||||
case LITERALS_RUN_LEN_V2:
|
||||
nCodingChoiceCost += 4;
|
||||
}
|
||||
else if (nNumLiterals == (LITERALS_RUN_LEN_V2 + 15)) {
|
||||
break;
|
||||
|
||||
case LITERALS_RUN_LEN_V2 + 15:
|
||||
nCodingChoiceCost += 8;
|
||||
}
|
||||
else if (nNumLiterals == 256) {
|
||||
break;
|
||||
|
||||
case 256:
|
||||
nCodingChoiceCost += 16;
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (nCodingChoiceCost < pDestLiteralSlots[nArrivalsPerPosition - 1].cost ||
|
||||
(nCodingChoiceCost == pDestLiteralSlots[nArrivalsPerPosition - 1].cost && nScore < (pDestLiteralSlots[nArrivalsPerPosition - 1].score) &&
|
||||
(nCodingChoiceCost == pDestLiteralSlots[nArrivalsPerPosition - 1].cost && nScore < pDestLiteralSlots[nArrivalsPerPosition - 1].score &&
|
||||
nRepOffset != pDestLiteralSlots[nArrivalsPerPosition - 1].rep_offset)) {
|
||||
int exists = 0, n;
|
||||
|
||||
|
@ -357,7 +365,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
|
||||
if (!exists) {
|
||||
for (;
|
||||
n < nArrivalsPerPosition && pDestLiteralSlots[n].cost == nCodingChoiceCost && nScore >= (pDestLiteralSlots[n].score);
|
||||
n < nArrivalsPerPosition && pDestLiteralSlots[n].cost == nCodingChoiceCost && nScore >= pDestLiteralSlots[n].score;
|
||||
n++) {
|
||||
if (pDestLiteralSlots[n].rep_offset == nRepOffset) {
|
||||
exists = 1;
|
||||
|
@ -392,11 +400,11 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
pDestArrival->cost = nCodingChoiceCost;
|
||||
pDestArrival->rep_offset = nRepOffset;
|
||||
pDestArrival->from_slot = j + 1;
|
||||
pDestArrival->from_pos = i;
|
||||
pDestArrival->from_pos = i - nStartOffset;
|
||||
pDestArrival->rep_len = cur_arrival[j].rep_len;
|
||||
pDestArrival->match_len = 0;
|
||||
pDestArrival->rep_pos = cur_arrival[j].rep_pos;
|
||||
pDestArrival->num_literals = nNumLiterals;
|
||||
pDestArrival->rep_pos = cur_arrival[j].rep_pos;
|
||||
pDestArrival->score = nScore + nDisableScore;
|
||||
}
|
||||
}
|
||||
|
@ -423,7 +431,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
const int nRepOffset = cur_arrival[j].rep_offset;
|
||||
|
||||
if (i >= nRepOffset) {
|
||||
if (!memcmp(pInWindow + i, pInWindow + i - nRepOffset, MIN_MATCH_SIZE_V2)) {
|
||||
if (!memcmp(pInWindowStart, pInWindowStart - nRepOffset, MIN_MATCH_SIZE_V2)) {
|
||||
if (nRepOffset) {
|
||||
const unsigned char* pInWindowAtPos;
|
||||
|
||||
|
@ -435,11 +443,11 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
nMinLen = nMaxRepLenForPos;
|
||||
pInWindowAtPos = pInWindowStart + nMinLen;
|
||||
|
||||
while ((pInWindowAtPos + 8) < pInWindowMax && !memcmp(pInWindowAtPos - nRepOffset, pInWindowAtPos, 8))
|
||||
while ((pInWindowAtPos + 8) < pInWindowMax && !memcmp(pInWindowAtPos, pInWindowAtPos - nRepOffset, 8))
|
||||
pInWindowAtPos += 8;
|
||||
while ((pInWindowAtPos + 4) < pInWindowMax && !memcmp(pInWindowAtPos - nRepOffset, pInWindowAtPos, 4))
|
||||
while ((pInWindowAtPos + 4) < pInWindowMax && !memcmp(pInWindowAtPos, pInWindowAtPos - nRepOffset, 4))
|
||||
pInWindowAtPos += 4;
|
||||
while (pInWindowAtPos < pInWindowMax && pInWindowAtPos[-nRepOffset] == pInWindowAtPos[0])
|
||||
while (pInWindowAtPos < pInWindowMax && pInWindowAtPos[0] == pInWindowAtPos[-nRepOffset])
|
||||
pInWindowAtPos++;
|
||||
const int nCurRepLen = (const int)(pInWindowAtPos - pInWindowStart);
|
||||
|
||||
|
@ -455,15 +463,14 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
nRepMatchArrivalIdxAndLen[nNumRepMatchArrivals] = -1;
|
||||
|
||||
if (!nReduce) {
|
||||
memset(nRepSlotHandledMask, 0, nArrivalsPerPosition * ((LCP_MAX + 1) / 8) * sizeof(char));
|
||||
memset(nRepSlotHandledMask, 0, nArrivalsPerPosition * ((LCP_MAX + 1) / 8) * sizeof(unsigned char));
|
||||
}
|
||||
memset(nRepLenHandledMask, 0, ((LCP_MAX + 1) / 8) * sizeof(char));
|
||||
memset(nRepLenHandledMask, 0, ((LCP_MAX + 1) / 8) * sizeof(unsigned char));
|
||||
|
||||
for (m = 0; m < NMATCHES_PER_INDEX_V2 && match[m].length; m++) {
|
||||
int nMatchLen = match[m].length & 0x7fff;
|
||||
int nMatchOffset = match[m].offset;
|
||||
int nNoRepmatchOffsetCost;
|
||||
int nNoRepmatchScore;
|
||||
const int nMatchOffset = match[m].offset;
|
||||
int nNoRepmatchOffsetCost = 0, nNoRepmatchScore = 0;
|
||||
int nStartingMatchLen, k;
|
||||
|
||||
if ((i + nMatchLen) > nEndOffset)
|
||||
|
@ -474,11 +481,9 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
|
||||
int nNonRepMatchArrivalIdx = -1;
|
||||
for (j = 0; j < nNumArrivalsForThisPos; j++) {
|
||||
const int nRepOffset = cur_arrival[j].rep_offset;
|
||||
|
||||
if (nMatchOffset != nRepOffset) {
|
||||
if (nMatchOffset != cur_arrival[j].rep_offset) {
|
||||
const int nPrevCost = cur_arrival[j].cost;
|
||||
const int nScorePenalty = 3 + ((match[m].length & 0x8000) >> 15);
|
||||
const int nScorePenalty = 3 + (match[m].length >> 15);
|
||||
|
||||
nNoRepmatchOffsetCost = nPrevCost /* the actual cost of the literals themselves accumulates up the chain */;
|
||||
if (!cur_arrival[j].num_literals)
|
||||
|
@ -498,7 +503,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
}
|
||||
else {
|
||||
nStartingMatchLen = nMinMatchSize;
|
||||
nMatchLenCost = 0 + 8 /* token */;
|
||||
nMatchLenCost = /* 0 + */ 8 /* token */;
|
||||
}
|
||||
|
||||
for (k = nStartingMatchLen; k <= nMatchLen; k++) {
|
||||
|
@ -514,7 +519,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
}
|
||||
}
|
||||
|
||||
lzsa_arrival *pDestSlots = &cur_arrival[k << ARRIVALS_PER_POSITION_SHIFT];
|
||||
lzsa_arrival *pDestSlots = &cur_arrival[k << ARRIVALS_PER_POSITION_SHIFT_V2];
|
||||
|
||||
/* Insert non-repmatch candidate */
|
||||
|
||||
|
@ -522,7 +527,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
const int nCodingChoiceCost = nMatchLenCost + nNoRepmatchOffsetCost;
|
||||
|
||||
if (nCodingChoiceCost < pDestSlots[nArrivalsPerPosition - 2].cost ||
|
||||
(nCodingChoiceCost == pDestSlots[nArrivalsPerPosition - 2].cost && nNoRepmatchScore < (pDestSlots[nArrivalsPerPosition - 2].score) &&
|
||||
(nCodingChoiceCost == pDestSlots[nArrivalsPerPosition - 2].cost && nNoRepmatchScore < pDestSlots[nArrivalsPerPosition - 2].score &&
|
||||
(nCodingChoiceCost != pDestSlots[nArrivalsPerPosition - 1].cost || nMatchOffset != pDestSlots[nArrivalsPerPosition - 1].rep_offset))) {
|
||||
int exists = 0, n;
|
||||
|
||||
|
@ -537,7 +542,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
|
||||
if (!exists) {
|
||||
for (;
|
||||
n < nArrivalsPerPosition && pDestSlots[n].cost == nCodingChoiceCost && nNoRepmatchScore >= (pDestSlots[n].score);
|
||||
n < nArrivalsPerPosition && pDestSlots[n].cost == nCodingChoiceCost && nNoRepmatchScore >= pDestSlots[n].score;
|
||||
n++) {
|
||||
if (pDestSlots[n].rep_offset == nMatchOffset) {
|
||||
exists = 1;
|
||||
|
@ -574,11 +579,11 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
pDestArrival->cost = nCodingChoiceCost;
|
||||
pDestArrival->rep_offset = nMatchOffset;
|
||||
pDestArrival->from_slot = nNonRepMatchArrivalIdx + 1;
|
||||
pDestArrival->from_pos = i;
|
||||
pDestArrival->from_pos = i - nStartOffset;
|
||||
pDestArrival->rep_len = k;
|
||||
pDestArrival->match_len = k;
|
||||
pDestArrival->rep_pos = i;
|
||||
pDestArrival->num_literals = 0;
|
||||
pDestArrival->rep_pos = i;
|
||||
pDestArrival->score = nNoRepmatchScore + nDisableScore;
|
||||
nRepLenHandledMask[k >> 3] &= ~((1 ^ nReduce) << (k & 7));
|
||||
}
|
||||
|
@ -608,7 +613,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
const int nRepCodingChoiceCost = nPrevCost /* the actual cost of the literals themselves accumulates up the chain */ + nMatchLenCost;
|
||||
|
||||
if (nRepCodingChoiceCost < pDestSlots[nArrivalsPerPosition - 1].cost ||
|
||||
(nRepCodingChoiceCost == pDestSlots[nArrivalsPerPosition - 1].cost && nScore < (pDestSlots[nArrivalsPerPosition - 1].score))) {
|
||||
(nRepCodingChoiceCost == pDestSlots[nArrivalsPerPosition - 1].cost && nScore < pDestSlots[nArrivalsPerPosition - 1].score)) {
|
||||
int exists = 0, n;
|
||||
|
||||
for (n = 0;
|
||||
|
@ -624,7 +629,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
|
||||
if (!exists) {
|
||||
for (;
|
||||
n < nArrivalsPerPosition && pDestSlots[n].cost == nRepCodingChoiceCost && nScore >= (pDestSlots[n].score);
|
||||
n < nArrivalsPerPosition && pDestSlots[n].cost == nRepCodingChoiceCost && nScore >= pDestSlots[n].score;
|
||||
n++) {
|
||||
if (pDestSlots[n].rep_offset == nRepOffset) {
|
||||
exists = 1;
|
||||
|
@ -659,11 +664,11 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
pDestArrival->cost = nRepCodingChoiceCost;
|
||||
pDestArrival->rep_offset = nRepOffset;
|
||||
pDestArrival->from_slot = j + 1;
|
||||
pDestArrival->from_pos = i;
|
||||
pDestArrival->from_pos = i - nStartOffset;
|
||||
pDestArrival->rep_len = k;
|
||||
pDestArrival->match_len = k;
|
||||
pDestArrival->rep_pos = i;
|
||||
pDestArrival->num_literals = 0;
|
||||
pDestArrival->rep_pos = i;
|
||||
pDestArrival->score = nScore + nDisableScore;
|
||||
nRepLenHandledMask[k >> 3] &= ~((1 ^ nReduce) << (k & 7));
|
||||
}
|
||||
|
@ -684,21 +689,19 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
}
|
||||
}
|
||||
|
||||
if (nMatchLen >= LCP_MAX && ((m + 1) >= NMATCHES_PER_INDEX_V2 || match[m + 1].length < LCP_MAX))
|
||||
if (nMatchLen >= LCP_MAX && ((m + 1) >= NMATCHES_PER_INDEX_V2 || (match[m + 1].length & 0x7fff) < LCP_MAX))
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!nInsertForwardReps) {
|
||||
const lzsa_arrival* end_arrival = &arrival[(i << ARRIVALS_PER_POSITION_SHIFT) + 0];
|
||||
const lzsa_arrival* end_arrival = &arrival[i << ARRIVALS_PER_POSITION_SHIFT_V2];
|
||||
lzsa_match* pBestMatch = pCompressor->best_match - nStartOffset;
|
||||
|
||||
while (end_arrival->from_slot > 0 && end_arrival->from_pos >= 0 && end_arrival->from_pos < nEndOffset) {
|
||||
pBestMatch[end_arrival->from_pos].length = end_arrival->match_len;
|
||||
if (end_arrival->match_len)
|
||||
pBestMatch[end_arrival->from_pos].offset = end_arrival->rep_offset;
|
||||
else
|
||||
pBestMatch[end_arrival->from_pos].offset = 0;
|
||||
end_arrival = &arrival[(end_arrival->from_pos << ARRIVALS_PER_POSITION_SHIFT) + (end_arrival->from_slot - 1)];
|
||||
while (end_arrival->from_slot > 0 && (end_arrival->from_pos + nStartOffset) < nEndOffset) {
|
||||
pBestMatch[end_arrival->from_pos + nStartOffset].length = end_arrival->match_len;
|
||||
pBestMatch[end_arrival->from_pos + nStartOffset].offset = (end_arrival->match_len) ? end_arrival->rep_offset : 0;
|
||||
end_arrival = &arrival[((end_arrival->from_pos + nStartOffset) << ARRIVALS_PER_POSITION_SHIFT_V2) + (end_arrival->from_slot - 1)];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -709,13 +712,13 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
*
|
||||
* @param pCompressor compression context
|
||||
* @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress)
|
||||
* @param pBestMatch optimal matches to evaluate and update
|
||||
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
|
||||
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
|
||||
*
|
||||
* @return non-zero if the number of tokens was reduced, 0 if it wasn't
|
||||
*/
|
||||
static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) {
|
||||
static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset) {
|
||||
lzsa_match* pBestMatch = pCompressor->best_match - nStartOffset;
|
||||
int i;
|
||||
int nNumLiterals = 0;
|
||||
int nPrevRepMatchOffset = 0;
|
||||
|
@ -759,28 +762,32 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const un
|
|||
nNextIndex++;
|
||||
}
|
||||
|
||||
if (nNextIndex < nEndOffset && pBestMatch[nNextIndex].length >= MIN_MATCH_SIZE_V2) {
|
||||
if (nNextIndex < nEndOffset) {
|
||||
/* This command is a match, is followed by 'nNextLiterals' literals and then by another match */
|
||||
|
||||
if (nRepMatchOffset && pMatch->offset != nRepMatchOffset && (pBestMatch[nNextIndex].offset != pMatch->offset || pBestMatch[nNextIndex].offset == nRepMatchOffset ||
|
||||
if (nRepMatchOffset && pMatch->offset != nRepMatchOffset && (pBestMatch[nNextIndex].offset != pMatch->offset ||
|
||||
((pMatch->offset <= 32) ? 4 : ((pMatch->offset <= 512) ? 8 : ((pMatch->offset <= (8192 + 512)) ? 12 : 16))) >
|
||||
((pBestMatch[nNextIndex].offset <= 32) ? 4 : ((pBestMatch[nNextIndex].offset <= 512) ? 8 : ((pBestMatch[nNextIndex].offset <= (8192 + 512)) ? 12 : 16))))) {
|
||||
/* Check if we can change the current match's offset to be the same as the previous match's offset, and get an extra repmatch. This will occur when
|
||||
* matching large regions of identical bytes for instance, where there are too many offsets to be considered by the parser, and when not compressing to favor the
|
||||
* ratio (the forward arrivals parser already has this covered). */
|
||||
if (i >= nRepMatchOffset &&
|
||||
(i - nRepMatchOffset + pMatch->length) <= nEndOffset &&
|
||||
!memcmp(pInWindow + i - nRepMatchOffset, pInWindow + i - pMatch->offset, pMatch->length)) {
|
||||
!memcmp(pInWindow + i - nRepMatchOffset, pInWindow + i, pMatch->length)) {
|
||||
pMatch->offset = nRepMatchOffset;
|
||||
nDidReduce = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (pBestMatch[nNextIndex].offset && pMatch->offset != pBestMatch[nNextIndex].offset && nRepMatchOffset != pBestMatch[nNextIndex].offset) {
|
||||
if (pBestMatch[nNextIndex].offset && pMatch->offset != pBestMatch[nNextIndex].offset) {
|
||||
/* Otherwise, try to gain a match forward as well */
|
||||
if (i >= pBestMatch[nNextIndex].offset && (i - pBestMatch[nNextIndex].offset + pMatch->length) <= nEndOffset) {
|
||||
if (i >= pBestMatch[nNextIndex].offset && (i + pMatch->length) <= nEndOffset) {
|
||||
int nMaxLen = 0;
|
||||
while (nMaxLen < pMatch->length && pInWindow[i - pBestMatch[nNextIndex].offset + nMaxLen] == pInWindow[i - pMatch->offset + nMaxLen])
|
||||
const unsigned char *pInWindowAtPos = pInWindow + i;
|
||||
while ((nMaxLen + 8) < pMatch->length && !memcmp(pInWindowAtPos + nMaxLen - pBestMatch[nNextIndex].offset, pInWindowAtPos + nMaxLen, 8))
|
||||
nMaxLen += 8;
|
||||
while ((nMaxLen + 4) < pMatch->length && !memcmp(pInWindowAtPos + nMaxLen - pBestMatch[nNextIndex].offset, pInWindowAtPos + nMaxLen, 4))
|
||||
nMaxLen += 4;
|
||||
while (nMaxLen < pMatch->length && pInWindowAtPos[nMaxLen - pBestMatch[nNextIndex].offset] == pInWindowAtPos[nMaxLen])
|
||||
nMaxLen++;
|
||||
if (nMaxLen >= pMatch->length) {
|
||||
/* Replace */
|
||||
|
@ -793,20 +800,24 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const un
|
|||
nPartialSizeBefore = lzsa_get_match_varlen_size_v2(pMatch->length - MIN_MATCH_SIZE_V2);
|
||||
nPartialSizeBefore += (pMatch->offset <= 32) ? 4 : ((pMatch->offset <= 512) ? 8 : ((pMatch->offset <= (8192 + 512)) ? 12 : 16));
|
||||
nPartialSizeBefore += lzsa_get_literals_varlen_size_v2(nNextLiterals);
|
||||
nPartialSizeBefore += (pBestMatch[nNextIndex].offset <= 32) ? 4 : ((pBestMatch[nNextIndex].offset <= 512) ? 8 : ((pBestMatch[nNextIndex].offset <= (8192 + 512)) ? 12 : 16));
|
||||
|
||||
nPartialSizeAfter = lzsa_get_match_varlen_size_v2(nMaxLen - MIN_MATCH_SIZE_V2);
|
||||
nPartialSizeAfter += lzsa_get_literals_varlen_size_v2(nNextLiterals + (pMatch->length - nMaxLen)) + ((pMatch->length - nMaxLen) << 3);
|
||||
if (nRepMatchOffset != pBestMatch[nNextIndex].offset)
|
||||
nPartialSizeAfter += (pBestMatch[nNextIndex].offset <= 32) ? 4 : ((pBestMatch[nNextIndex].offset <= 512) ? 8 : ((pBestMatch[nNextIndex].offset <= (8192 + 512)) ? 12 : 16));
|
||||
|
||||
if (nPartialSizeAfter < nPartialSizeBefore) {
|
||||
const int nMatchLen = pMatch->length;
|
||||
int j;
|
||||
|
||||
/* We gain a repmatch that is shorter than the original match as this is the best we can do, so it is followed by extra literals, but
|
||||
* we have calculated that this is shorter */
|
||||
pMatch->length = nMaxLen;
|
||||
pMatch->offset = pBestMatch[nNextIndex].offset;
|
||||
for (j = nMaxLen; j < pMatch->length; j++) {
|
||||
for (j = nMaxLen; j < nMatchLen; j++) {
|
||||
pBestMatch[i + j].length = 0;
|
||||
}
|
||||
pMatch->length = nMaxLen;
|
||||
nDidReduce = 1;
|
||||
}
|
||||
}
|
||||
|
@ -862,8 +873,8 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const un
|
|||
}
|
||||
}
|
||||
|
||||
if ((i + pMatch->length) < nEndOffset && pMatch->offset > 0 && pMatch->length >= MIN_MATCH_SIZE_V2 &&
|
||||
pBestMatch[i + pMatch->length].offset > 0 &&
|
||||
if ((i + pMatch->length) < nEndOffset && pMatch->offset && pMatch->length >= MIN_MATCH_SIZE_V2 &&
|
||||
pBestMatch[i + pMatch->length].offset &&
|
||||
pBestMatch[i + pMatch->length].length >= MIN_MATCH_SIZE_V2 &&
|
||||
(pMatch->length + pBestMatch[i + pMatch->length].length) <= MAX_VARLEN &&
|
||||
(i + pMatch->length) >= pMatch->offset &&
|
||||
|
@ -911,8 +922,8 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const un
|
|||
}
|
||||
|
||||
nPrevRepMatchOffset = nRepMatchOffset;
|
||||
nRepMatchOffset = pMatch->offset;
|
||||
nRepMatchLen = pMatch->length;
|
||||
nRepMatchOffset = pMatch->offset;
|
||||
nRepIndex = i;
|
||||
|
||||
i += pMatch->length;
|
||||
|
@ -931,7 +942,6 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const un
|
|||
* Emit block of compressed data
|
||||
*
|
||||
* @param pCompressor compression context
|
||||
* @param pBestMatch optimal matches to emit
|
||||
* @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress)
|
||||
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
|
||||
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
|
||||
|
@ -940,7 +950,8 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const un
|
|||
*
|
||||
* @return size of compressed data in output buffer, or -1 if the data is uncompressible
|
||||
*/
|
||||
static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const lzsa_match *pBestMatch, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) {
|
||||
static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) {
|
||||
const lzsa_match *pBestMatch = pCompressor->best_match - nStartOffset;
|
||||
int i;
|
||||
int nNumLiterals = 0;
|
||||
int nInFirstLiteralOffset = 0;
|
||||
|
@ -952,8 +963,8 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const lzsa_match *p
|
|||
const lzsa_match *pMatch = pBestMatch + i;
|
||||
|
||||
if (pMatch->length >= MIN_MATCH_SIZE_V2) {
|
||||
const int nMatchOffset = pMatch->offset;
|
||||
const int nMatchLen = pMatch->length;
|
||||
const int nMatchOffset = pMatch->offset;
|
||||
const int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE_V2;
|
||||
const int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V2) ? LITERALS_RUN_LEN_V2 : nNumLiterals;
|
||||
const int nTokenMatchLen = (nEncodedMatchLen >= MATCH_RUN_LEN_V2) ? MATCH_RUN_LEN_V2 : nEncodedMatchLen;
|
||||
|
@ -966,7 +977,7 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const lzsa_match *p
|
|||
}
|
||||
else {
|
||||
if (nMatchOffset <= 32) {
|
||||
nTokenOffsetMode = 0x00 | ((((-nMatchOffset) & 0x01) << 5) ^ 0x20);
|
||||
nTokenOffsetMode = /* 0x00 | */ ((((-nMatchOffset) & 0x01) << 5) ^ 0x20);
|
||||
nOffsetSize = 4;
|
||||
}
|
||||
else if (nMatchOffset <= 512) {
|
||||
|
@ -1065,7 +1076,7 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const lzsa_match *p
|
|||
i += nMatchLen;
|
||||
|
||||
if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
|
||||
int nCurSafeDist = (i - nStartOffset) - nOutOffset;
|
||||
const int nCurSafeDist = (i - nStartOffset) - nOutOffset;
|
||||
if (nCurSafeDist >= 0 && pCompressor->safe_dist < nCurSafeDist)
|
||||
pCompressor->safe_dist = nCurSafeDist;
|
||||
}
|
||||
|
@ -1090,7 +1101,7 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const lzsa_match *p
|
|||
if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK)
|
||||
pOutData[nOutOffset++] = (nTokenLiteralsLen << 3) | 0xe7;
|
||||
else
|
||||
pOutData[nOutOffset++] = (nTokenLiteralsLen << 3) | 0x00;
|
||||
pOutData[nOutOffset++] = (nTokenLiteralsLen << 3) /* | 0x00 */;
|
||||
nOutOffset = lzsa_write_literals_varlen_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, nNumLiterals);
|
||||
if (nOutOffset < 0) return -1;
|
||||
|
||||
|
@ -1104,11 +1115,10 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const lzsa_match *p
|
|||
if (nNumLiterals != 0) {
|
||||
memcpy(pOutData + nOutOffset, pInWindow + nInFirstLiteralOffset, nNumLiterals);
|
||||
nOutOffset += nNumLiterals;
|
||||
nNumLiterals = 0;
|
||||
}
|
||||
|
||||
if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
|
||||
int nCurSafeDist = (i - nStartOffset) - nOutOffset;
|
||||
const int nCurSafeDist = (i - nStartOffset) - nOutOffset;
|
||||
if (nCurSafeDist >= 0 && pCompressor->safe_dist < nCurSafeDist)
|
||||
pCompressor->safe_dist = nCurSafeDist;
|
||||
}
|
||||
|
@ -1154,7 +1164,7 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const lzsa_match *p
|
|||
*/
|
||||
static int lzsa_write_raw_uncompressed_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) {
|
||||
int nCurNibbleOffset = -1;
|
||||
int nNumLiterals = nEndOffset - nStartOffset;
|
||||
const int nNumLiterals = nEndOffset - nStartOffset;
|
||||
const int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V2) ? LITERALS_RUN_LEN_V2 : nNumLiterals;
|
||||
int nOutOffset = 0;
|
||||
|
||||
|
@ -1171,7 +1181,6 @@ static int lzsa_write_raw_uncompressed_block_v2(lzsa_compressor *pCompressor, co
|
|||
if (nNumLiterals != 0) {
|
||||
memcpy(pOutData + nOutOffset, pInWindow + nStartOffset, nNumLiterals);
|
||||
nOutOffset += nNumLiterals;
|
||||
nNumLiterals = 0;
|
||||
}
|
||||
|
||||
/* Emit EOD marker for raw block */
|
||||
|
@ -1231,9 +1240,7 @@ int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
/* Compress optimally without breaking ties in favor of less tokens */
|
||||
|
||||
memset(pCompressor->best_match, 0, BLOCK_SIZE * sizeof(lzsa_match));
|
||||
lzsa_optimize_forward_v2(pCompressor, pInWindow, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 0 /* reduce */, (nInDataSize < 65536) ? 1 : 0 /* insert forward reps */, nArrivalsPerPosition);
|
||||
|
||||
lzsa_match* pBestMatch = pCompressor->best_match - nPreviousBlockSize;
|
||||
lzsa_optimize_forward_v2(pCompressor, pInWindow, nPreviousBlockSize, nEndOffset, 0 /* reduce */, (nInDataSize < 65536) ? 1 : 0 /* insert forward reps */, nArrivalsPerPosition);
|
||||
|
||||
if (nInDataSize < 65536) {
|
||||
int* first_offset_for_byte = pCompressor->first_offset_for_byte;
|
||||
|
@ -1449,22 +1456,20 @@ int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigne
|
|||
}
|
||||
|
||||
/* Compress optimally and do break ties in favor of less tokens */
|
||||
lzsa_optimize_forward_v2(pCompressor, pInWindow, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 1 /* reduce */, 0 /* use forward reps */, 1 << ARRIVALS_PER_POSITION_SHIFT);
|
||||
|
||||
pBestMatch = pCompressor->best_match - nPreviousBlockSize;
|
||||
lzsa_optimize_forward_v2(pCompressor, pInWindow, nPreviousBlockSize, nEndOffset, 1 /* reduce */, 0 /* use forward reps */, NARRIVALS_PER_POSITION_V2_MAX);
|
||||
}
|
||||
|
||||
/* Try to reduce final command set, wherever possible */
|
||||
nPasses = 0;
|
||||
do {
|
||||
nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pInWindow, pBestMatch, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
|
||||
nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pInWindow, nPreviousBlockSize, nEndOffset);
|
||||
nPasses++;
|
||||
} while (nDidReduce && nPasses < 20);
|
||||
|
||||
/* Write compressed block */
|
||||
nResult = lzsa_write_block_v2(pCompressor, pBestMatch, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
|
||||
if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
|
||||
nResult = lzsa_write_raw_uncompressed_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
|
||||
nResult = lzsa_write_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nEndOffset, pOutData, nMaxOutDataSize);
|
||||
if (nResult < 0 && (pCompressor->flags & LZSA_FLAG_RAW_BLOCK)) {
|
||||
nResult = lzsa_write_raw_uncompressed_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nEndOffset, pOutData, nMaxOutDataSize);
|
||||
}
|
||||
|
||||
return nResult;
|
||||
|
|
|
@ -33,8 +33,11 @@
|
|||
#ifndef _SHRINK_BLOCK_V2_H
|
||||
#define _SHRINK_BLOCK_V2_H
|
||||
|
||||
/* Forward declarations */
|
||||
typedef struct _lzsa_compressor lzsa_compressor;
|
||||
#include "shrink_context.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Select the most optimal matches, reduce the token count if possible, and then emit a block of compressed LZSA2 data
|
||||
|
@ -50,4 +53,8 @@ typedef struct _lzsa_compressor lzsa_compressor;
|
|||
*/
|
||||
int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _SHRINK_BLOCK_V2_H */
|
||||
|
|
|
@ -45,14 +45,15 @@
|
|||
* @param pCompressor compression context to initialize
|
||||
* @param nMaxWindowSize maximum size of input data window (previously compressed bytes + bytes to compress)
|
||||
* @param nMinMatchSize minimum match size (cannot be less than MIN_MATCH_SIZE)
|
||||
* @param nFormatVersion version of format to use (1-2)
|
||||
* @param nFlags compression flags
|
||||
*
|
||||
* @return 0 for success, non-zero for failure
|
||||
*/
|
||||
int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize, const int nMinMatchSize, const int nFormatVersion, const int nFlags) {
|
||||
int nResult;
|
||||
int nMinMatchSizeForFormat = (nFormatVersion == 1) ? MIN_MATCH_SIZE_V1 : MIN_MATCH_SIZE_V2;
|
||||
int nMaxMinMatchForFormat = (nFormatVersion == 1) ? 5 : 3;
|
||||
const int nMinMatchSizeForFormat = (nFormatVersion == 1) ? MIN_MATCH_SIZE_V1 : MIN_MATCH_SIZE_V2;
|
||||
const int nMaxMinMatchForFormat = (nFormatVersion == 1) ? 5 : 3;
|
||||
|
||||
nResult = divsufsort_init(&pCompressor->divsufsort_context);
|
||||
pCompressor->intervals = NULL;
|
||||
|
@ -60,7 +61,6 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
|
|||
pCompressor->open_intervals = NULL;
|
||||
pCompressor->match = NULL;
|
||||
pCompressor->best_match = NULL;
|
||||
pCompressor->improved_match = NULL;
|
||||
pCompressor->arrival = NULL;
|
||||
pCompressor->rep_slot_handled_mask = NULL;
|
||||
pCompressor->rep_len_handled_mask = NULL;
|
||||
|
@ -94,41 +94,37 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
|
|||
pCompressor->open_intervals = (unsigned int *)malloc((LCP_AND_TAG_MAX + 1) * sizeof(unsigned int));
|
||||
|
||||
if (pCompressor->open_intervals) {
|
||||
pCompressor->arrival = (lzsa_arrival *)malloc(((BLOCK_SIZE + 1) << ARRIVALS_PER_POSITION_SHIFT) * sizeof(lzsa_arrival));
|
||||
pCompressor->arrival = (lzsa_arrival *)malloc(((BLOCK_SIZE + 1) << ARRIVALS_PER_POSITION_SHIFT_V2) * sizeof(lzsa_arrival));
|
||||
|
||||
if (pCompressor->arrival) {
|
||||
pCompressor->best_match = (lzsa_match *)malloc(BLOCK_SIZE * sizeof(lzsa_match));
|
||||
|
||||
if (pCompressor->best_match) {
|
||||
pCompressor->improved_match = (lzsa_match *)malloc(BLOCK_SIZE * sizeof(lzsa_match));
|
||||
|
||||
if (pCompressor->improved_match) {
|
||||
if (pCompressor->format_version == 2)
|
||||
pCompressor->match = (lzsa_match *)malloc(BLOCK_SIZE * NMATCHES_PER_INDEX_V2 * sizeof(lzsa_match));
|
||||
else
|
||||
pCompressor->match = (lzsa_match *)malloc(BLOCK_SIZE * NMATCHES_PER_INDEX_V1 * sizeof(lzsa_match));
|
||||
if (pCompressor->match) {
|
||||
if (pCompressor->format_version == 2) {
|
||||
pCompressor->rep_slot_handled_mask = (char*)malloc(NARRIVALS_PER_POSITION_V2_BIG * ((LCP_MAX + 1) / 8) * sizeof(char));
|
||||
if (pCompressor->rep_slot_handled_mask) {
|
||||
pCompressor->rep_len_handled_mask = (char*)malloc(((LCP_MAX + 1) / 8) * sizeof(char));
|
||||
if (pCompressor->rep_len_handled_mask) {
|
||||
pCompressor->first_offset_for_byte = (int*)malloc(65536 * sizeof(int));
|
||||
if (pCompressor->first_offset_for_byte) {
|
||||
pCompressor->next_offset_for_pos = (int*)malloc(BLOCK_SIZE * sizeof(int));
|
||||
if (pCompressor->next_offset_for_pos) {
|
||||
pCompressor->offset_cache = (int*)malloc(2048 * sizeof(int));
|
||||
if (pCompressor->offset_cache) {
|
||||
return 0;
|
||||
}
|
||||
if (pCompressor->format_version == 2)
|
||||
pCompressor->match = (lzsa_match*)malloc(BLOCK_SIZE * NMATCHES_PER_INDEX_V2 * sizeof(lzsa_match));
|
||||
else
|
||||
pCompressor->match = (lzsa_match*)malloc(BLOCK_SIZE * NMATCHES_PER_INDEX_V1 * sizeof(lzsa_match));
|
||||
if (pCompressor->match) {
|
||||
if (pCompressor->format_version == 2) {
|
||||
pCompressor->rep_slot_handled_mask = (unsigned char*)malloc(NARRIVALS_PER_POSITION_V2_BIG * ((LCP_MAX + 1) / 8) * sizeof(unsigned char));
|
||||
if (pCompressor->rep_slot_handled_mask) {
|
||||
pCompressor->rep_len_handled_mask = (unsigned char*)malloc(((LCP_MAX + 1) / 8) * sizeof(unsigned char));
|
||||
if (pCompressor->rep_len_handled_mask) {
|
||||
pCompressor->first_offset_for_byte = (int*)malloc(65536 * sizeof(int));
|
||||
if (pCompressor->first_offset_for_byte) {
|
||||
pCompressor->next_offset_for_pos = (int*)malloc(BLOCK_SIZE * sizeof(int));
|
||||
if (pCompressor->next_offset_for_pos) {
|
||||
pCompressor->offset_cache = (int*)malloc(2048 * sizeof(int));
|
||||
if (pCompressor->offset_cache) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -180,11 +176,6 @@ void lzsa_compressor_destroy(lzsa_compressor *pCompressor) {
|
|||
pCompressor->match = NULL;
|
||||
}
|
||||
|
||||
if (pCompressor->improved_match) {
|
||||
free(pCompressor->improved_match);
|
||||
pCompressor->improved_match = NULL;
|
||||
}
|
||||
|
||||
if (pCompressor->arrival) {
|
||||
free(pCompressor->arrival);
|
||||
pCompressor->arrival = NULL;
|
||||
|
|
|
@ -53,7 +53,8 @@ extern "C" {
|
|||
#define NARRIVALS_PER_POSITION_V2_SMALL 9
|
||||
#define NARRIVALS_PER_POSITION_V2_BIG 32
|
||||
#define NARRIVALS_PER_POSITION_V2_MAX 64
|
||||
#define ARRIVALS_PER_POSITION_SHIFT 6
|
||||
#define ARRIVALS_PER_POSITION_SHIFT_V1 3
|
||||
#define ARRIVALS_PER_POSITION_SHIFT_V2 6
|
||||
|
||||
#define NMATCHES_PER_INDEX_V1 16
|
||||
#define MATCHES_PER_INDEX_SHIFT_V1 4
|
||||
|
@ -78,11 +79,11 @@ typedef struct _lzsa_arrival {
|
|||
unsigned short rep_offset;
|
||||
short from_slot;
|
||||
|
||||
int from_pos;
|
||||
unsigned short from_pos;
|
||||
unsigned short rep_len;
|
||||
unsigned short match_len;
|
||||
unsigned short num_literals;
|
||||
int rep_pos;
|
||||
int num_literals;
|
||||
int score;
|
||||
} lzsa_arrival;
|
||||
|
||||
|
@ -123,10 +124,9 @@ typedef struct _lzsa_compressor {
|
|||
unsigned int *open_intervals;
|
||||
lzsa_match *match;
|
||||
lzsa_match *best_match;
|
||||
lzsa_match *improved_match;
|
||||
lzsa_arrival *arrival;
|
||||
char *rep_slot_handled_mask;
|
||||
char *rep_len_handled_mask;
|
||||
unsigned char *rep_slot_handled_mask;
|
||||
unsigned char *rep_len_handled_mask;
|
||||
int *first_offset_for_byte;
|
||||
int *next_offset_for_pos;
|
||||
int *offset_cache;
|
||||
|
@ -144,6 +144,7 @@ typedef struct _lzsa_compressor {
|
|||
* @param pCompressor compression context to initialize
|
||||
* @param nMaxWindowSize maximum size of input data window (previously compressed bytes + bytes to compress)
|
||||
* @param nMinMatchSize minimum match size (cannot be less than MIN_MATCH_SIZE)
|
||||
* @param nFormatVersion version of format to use (1-2)
|
||||
* @param nFlags compression flags
|
||||
*
|
||||
* @return 0 for success, non-zero for failure
|
||||
|
|
|
@ -33,16 +33,13 @@
|
|||
#ifndef _SHRINK_STREAMING_H
|
||||
#define _SHRINK_STREAMING_H
|
||||
|
||||
#include "shrink_context.h"
|
||||
#include "stream.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Forward declaration */
|
||||
typedef enum _lzsa_status_t lzsa_status_t;
|
||||
typedef struct _lzsa_stats lzsa_stats;
|
||||
|
||||
/*-------------- File API -------------- */
|
||||
|
||||
/**
|
||||
|
|
10
src/stream.c
10
src/stream.c
|
@ -44,7 +44,7 @@
|
|||
*
|
||||
* @param stream stream
|
||||
*/
|
||||
static void lzsa_filestream_close(lzsa_stream_t *stream) {
|
||||
static void lzsa_filestream_close(struct _lzsa_stream_t *stream) {
|
||||
if (stream->obj) {
|
||||
fclose((FILE*)stream->obj);
|
||||
stream->obj = NULL;
|
||||
|
@ -64,7 +64,7 @@ static void lzsa_filestream_close(lzsa_stream_t *stream) {
|
|||
*
|
||||
* @return number of bytes read
|
||||
*/
|
||||
static size_t lzsa_filestream_read(lzsa_stream_t *stream, void *ptr, size_t size) {
|
||||
static size_t lzsa_filestream_read(struct _lzsa_stream_t *stream, void *ptr, size_t size) {
|
||||
return fread(ptr, 1, size, (FILE*)stream->obj);
|
||||
}
|
||||
|
||||
|
@ -77,7 +77,7 @@ static size_t lzsa_filestream_read(lzsa_stream_t *stream, void *ptr, size_t size
|
|||
*
|
||||
* @return number of bytes written
|
||||
*/
|
||||
static size_t lzsa_filestream_write(lzsa_stream_t *stream, void *ptr, size_t size) {
|
||||
static size_t lzsa_filestream_write(struct _lzsa_stream_t *stream, void *ptr, size_t size) {
|
||||
return fwrite(ptr, 1, size, (FILE*)stream->obj);
|
||||
}
|
||||
|
||||
|
@ -88,7 +88,7 @@ static size_t lzsa_filestream_write(lzsa_stream_t *stream, void *ptr, size_t siz
|
|||
*
|
||||
* @return nonzero if the end of the data has been reached, 0 if there is more data
|
||||
*/
|
||||
static int lzsa_filestream_eof(lzsa_stream_t *stream) {
|
||||
static int lzsa_filestream_eof(struct _lzsa_stream_t *stream) {
|
||||
return feof((FILE*)stream->obj);
|
||||
}
|
||||
|
||||
|
@ -105,7 +105,9 @@ int lzsa_filestream_open(lzsa_stream_t *stream, const char *pszInFilename, const
|
|||
const char* stdInOutFile = "-";
|
||||
const char* stdInMode = "rb";
|
||||
const char* stdOutMode = "wb";
|
||||
#ifdef _WIN32
|
||||
int result;
|
||||
#endif
|
||||
|
||||
if (!strncmp(pszInFilename, stdInOutFile, 1)) {
|
||||
if (!strncmp(pszMode, stdInMode, 2)) {
|
||||
|
|
27
src/stream.h
27
src/stream.h
|
@ -37,8 +37,23 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Forward declaration */
|
||||
typedef struct _lzsa_stream_t lzsa_stream_t;
|
||||
/** High level status for compression and decompression */
|
||||
typedef enum _lzsa_status_t {
|
||||
LZSA_OK = 0, /**< Success */
|
||||
LZSA_ERROR_SRC, /**< Error reading input */
|
||||
LZSA_ERROR_DST, /**< Error reading output */
|
||||
LZSA_ERROR_DICTIONARY, /**< Error reading dictionary */
|
||||
LZSA_ERROR_MEMORY, /**< Out of memory */
|
||||
|
||||
/* Compression-specific status codes */
|
||||
LZSA_ERROR_COMPRESSION, /**< Internal compression error */
|
||||
LZSA_ERROR_RAW_TOOLARGE, /**< Input is too large to be compressed to a raw block */
|
||||
LZSA_ERROR_RAW_UNCOMPRESSED, /**< Input is incompressible and raw blocks don't support uncompressed data */
|
||||
|
||||
/* Decompression-specific status codes */
|
||||
LZSA_ERROR_FORMAT, /**< Invalid input format or magic number when decompressing */
|
||||
LZSA_ERROR_DECOMPRESSION /**< Internal decompression error */
|
||||
} lzsa_status_t;
|
||||
|
||||
/* I/O stream */
|
||||
typedef struct _lzsa_stream_t {
|
||||
|
@ -54,7 +69,7 @@ typedef struct _lzsa_stream_t {
|
|||
*
|
||||
* @return number of bytes read
|
||||
*/
|
||||
size_t(*read)(lzsa_stream_t *stream, void *ptr, size_t size);
|
||||
size_t(*read)(struct _lzsa_stream_t *stream, void *ptr, size_t size);
|
||||
|
||||
/**
|
||||
* Write to stream
|
||||
|
@ -65,7 +80,7 @@ typedef struct _lzsa_stream_t {
|
|||
*
|
||||
* @return number of bytes written
|
||||
*/
|
||||
size_t(*write)(lzsa_stream_t *stream, void *ptr, size_t size);
|
||||
size_t(*write)(struct _lzsa_stream_t *stream, void *ptr, size_t size);
|
||||
|
||||
|
||||
/**
|
||||
|
@ -75,14 +90,14 @@ typedef struct _lzsa_stream_t {
|
|||
*
|
||||
* @return nonzero if the end of the data has been reached, 0 if there is more data
|
||||
*/
|
||||
int(*eof)(lzsa_stream_t *stream);
|
||||
int(*eof)(struct _lzsa_stream_t *stream);
|
||||
|
||||
/**
|
||||
* Close stream
|
||||
*
|
||||
* @param stream stream
|
||||
*/
|
||||
void(*close)(lzsa_stream_t *stream);
|
||||
void(*close)(struct _lzsa_stream_t *stream);
|
||||
} lzsa_stream_t;
|
||||
|
||||
/**
|
||||
|
|
Loading…
Reference in New Issue
Block a user