From a0ac24d105f20e3510bf3e62f9592b756d94f8b6 Mon Sep 17 00:00:00 2001 From: Emmanuel Marty Date: Sun, 15 Dec 2019 23:36:51 +0100 Subject: [PATCH] Tiny LZSA1 ratio increase; small improvements --- src/matchfinder.c | 5 +---- src/shrink_block_v1.c | 14 +++++++------- src/shrink_block_v2.c | 13 +++++++++---- src/shrink_context.h | 6 ++++-- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/matchfinder.c b/src/matchfinder.c index 144d347..fbdc5ca 100644 --- a/src/matchfinder.c +++ b/src/matchfinder.c @@ -105,7 +105,7 @@ int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *p } } else { - for (i = 1; i < nInWindowSize - 1; i++) { + for (i = 1; i < nInWindowSize; i++) { int nIndex = (int)(intervals[i] & POS_MASK); int nLen = PLCP[nIndex]; if (nLen < nMinMatchSize) @@ -116,9 +116,6 @@ int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *p } } - if (i < nInWindowSize) - intervals[i] &= POS_MASK; - /** * Build intervals for finding matches * diff --git a/src/shrink_block_v1.c b/src/shrink_block_v1.c index efc7c09..c30e4a9 100644 --- a/src/shrink_block_v1.c +++ b/src/shrink_block_v1.c @@ -172,7 +172,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p for (i = nStartOffset; i != nEndOffset; i++) { int m; - for (j = 0; j < NMATCHES_PER_ARRIVAL_SMALL && arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].from_slot; j++) { + for (j = 0; j < NMATCHES_PER_ARRIVAL_V1 && arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].from_slot; j++) { int nPrevCost = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].cost; int nCodingChoiceCost = nPrevCost + 8 /* literal */; int nScore = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].score + 1; @@ -185,7 +185,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p if (!nFavorRatio && nNumLiterals == 1) nCodingChoiceCost += MODESWITCH_PENALTY; - for (n = 0; n < NMATCHES_PER_ARRIVAL_SMALL /* we only need the literals + short match cost + long match cost cases */; n++) { + for (n = 0; n < NMATCHES_PER_ARRIVAL_V1 /* we only need the literals + short match cost + long match cost cases */; n++) { lzsa_arrival *pDestArrival = &arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n]; if (pDestArrival->from_slot == 0 || @@ -193,7 +193,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p (nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) { memmove(&arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n + 1], &arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n], - sizeof(lzsa_arrival) * (NMATCHES_PER_ARRIVAL_SMALL - n - 1)); + sizeof(lzsa_arrival) * (NMATCHES_PER_ARRIVAL_V1 - n - 1)); pDestArrival->cost = nCodingChoiceCost; pDestArrival->from_pos = i; @@ -225,7 +225,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p for (k = nStartingMatchLen; k <= nMatchLen; k++) { int nMatchLenCost = lzsa_get_match_varlen_size_v1(k - MIN_MATCH_SIZE_V1); - for (j = 0; j < NMATCHES_PER_ARRIVAL_SMALL && arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].from_slot; j++) { + for (j = 0; j < NMATCHES_PER_ARRIVAL_V1 && arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].from_slot; j++) { int nPrevCost = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].cost; int nCodingChoiceCost = nPrevCost + 8 /* token */ /* the actual cost of the literals themselves accumulates up the chain */ + nMatchOffsetCost + nMatchLenCost; int nScore = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].score + 5; @@ -235,7 +235,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p nCodingChoiceCost += MODESWITCH_PENALTY; for (n = 0; - n < NMATCHES_PER_ARRIVAL_SMALL && arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].from_slot && arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].cost <= nCodingChoiceCost; + n < NMATCHES_PER_ARRIVAL_V1 && arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].from_slot && arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].cost <= nCodingChoiceCost; n++) { if (lzsa_get_offset_cost_v1(arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].rep_offset) == lzsa_get_offset_cost_v1(match[m].offset)) { exists = 1; @@ -243,7 +243,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p } } - for (n = 0; !exists && n < NMATCHES_PER_ARRIVAL_SMALL /* we only need the literals + short match cost + long match cost cases */; n++) { + for (n = 0; !exists && n < NMATCHES_PER_ARRIVAL_V1 /* we only need the literals + short match cost + long match cost cases */; n++) { lzsa_arrival *pDestArrival = &arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n]; if (pDestArrival->from_slot == 0 || @@ -251,7 +251,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p (nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) { memmove(&arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n + 1], &arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n], - sizeof(lzsa_arrival) * (NMATCHES_PER_ARRIVAL_SMALL - n - 1)); + sizeof(lzsa_arrival) * (NMATCHES_PER_ARRIVAL_V1 - n - 1)); pDestArrival->cost = nCodingChoiceCost; pDestArrival->from_pos = i; diff --git a/src/shrink_block_v2.c b/src/shrink_block_v2.c index 9192785..7b2febc 100644 --- a/src/shrink_block_v2.c +++ b/src/shrink_block_v2.c @@ -183,6 +183,7 @@ static inline int lzsa_write_match_varlen_v2(unsigned char *pOutData, int nOutOf * @param nMatchOffset match offset to use as rep candidate * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * @param nMatchesPerArrival number of arrivals to record per input buffer position * @param nDepth current insertion depth */ static void lzsa_insert_forward_match_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int i, const int nMatchOffset, const int nStartOffset, const int nEndOffset, const int nMatchesPerArrival, int nDepth) { @@ -241,15 +242,19 @@ static void lzsa_insert_forward_match_v2(lzsa_compressor *pCompressor, const uns * * @param pCompressor compression context * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param pBestMatch pointer to buffer for outputting optimal matches * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * @param nReduce non-zero to reduce the number of tokens when the path costs are equal, zero not to * @param nInsertForwardReps non-zero to insert forward repmatch candidates, zero to use the previously inserted candidates + * @param nMatchesPerArrival number of arrivals to record per input buffer position */ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset, const int nReduce, const int nInsertForwardReps, const int nMatchesPerArrival) { lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << MATCHES_PER_ARRIVAL_SHIFT); const int nFavorRatio = (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) ? 1 : 0; const int nMinMatchSize = pCompressor->min_match_size; const int nDisableScore = nReduce ? 0 : (2 * BLOCK_SIZE); + const int nLeaveAloneMatchSize = (nMatchesPerArrival == NMATCHES_PER_ARRIVAL_V2_SMALL) ? LEAVE_ALONE_MATCH_SIZE_SMALL : LEAVE_ALONE_MATCH_SIZE; int i, j, n; if ((nEndOffset - nStartOffset) > BLOCK_SIZE) return; @@ -334,7 +339,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne lzsa_match *match = pCompressor->match + ((i - nStartOffset) << MATCHES_PER_INDEX_SHIFT_V2); - int nMinRepLen[NMATCHES_PER_ARRIVAL_BIG]; + int nMinRepLen[NMATCHES_PER_ARRIVAL_V2_BIG]; memset(nMinRepLen, 0, nMatchesPerArrival * sizeof(int)); for (m = 0; m < NMATCHES_PER_INDEX_V2 && match[m].length; m++) { @@ -343,7 +348,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne int nScorePenalty = ((match[m].length & 0x8000) >> 15); int nNoRepmatchOffsetCost = (nMatchOffset <= 32) ? 4 : ((nMatchOffset <= 512) ? 8 : ((nMatchOffset <= (8192 + 512)) ? 12 : 16)); int nStartingMatchLen, k; - int nMaxRepLen[NMATCHES_PER_ARRIVAL_BIG]; + int nMaxRepLen[NMATCHES_PER_ARRIVAL_V2_BIG]; if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS)) nMatchLen = nEndOffset - LAST_LITERALS - i; @@ -375,7 +380,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne lzsa_insert_forward_match_v2(pCompressor, pInWindow, i, nMatchOffset, nStartOffset, nEndOffset, nMatchesPerArrival, 0); int nMatchLenCost = 0; - if (nMatchLen >= LEAVE_ALONE_MATCH_SIZE) { + if (nMatchLen >= nLeaveAloneMatchSize) { nStartingMatchLen = nMatchLen; nMatchLenCost = 4 + 24; } @@ -1124,7 +1129,7 @@ static int lzsa_write_raw_uncompressed_block_v2(lzsa_compressor *pCompressor, co */ int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) { int nResult, nBaseCompressedSize; - int nMatchesPerArrival = (nInDataSize < 65536) ? NMATCHES_PER_ARRIVAL_BIG : NMATCHES_PER_ARRIVAL_SMALL; + int nMatchesPerArrival = (nInDataSize < 65536) ? NMATCHES_PER_ARRIVAL_V2_BIG : NMATCHES_PER_ARRIVAL_V2_SMALL; /* Compress optimally without breaking ties in favor of less tokens */ diff --git a/src/shrink_context.h b/src/shrink_context.h index d2916c9..70245cf 100644 --- a/src/shrink_context.h +++ b/src/shrink_context.h @@ -49,8 +49,9 @@ extern "C" { #define VISITED_FLAG 0x80000000 #define EXCL_VISITED_MASK 0x7fffffff -#define NMATCHES_PER_ARRIVAL_SMALL 8 -#define NMATCHES_PER_ARRIVAL_BIG 32 +#define NMATCHES_PER_ARRIVAL_V1 8 +#define NMATCHES_PER_ARRIVAL_V2_SMALL 9 +#define NMATCHES_PER_ARRIVAL_V2_BIG 32 #define MATCHES_PER_ARRIVAL_SHIFT 5 #define NMATCHES_PER_INDEX_V1 8 @@ -60,6 +61,7 @@ extern "C" { #define MATCHES_PER_INDEX_SHIFT_V2 6 #define LEAVE_ALONE_MATCH_SIZE 300 +#define LEAVE_ALONE_MATCH_SIZE_SMALL 1000 #define LAST_LITERALS 0