6502/lzsa
1
0
Fork 0
mirror of https://github.com/emmanuel-marty/lzsa.git synced 2024-11-26 02:49:19 +00:00
Code Issues Projects Releases Wiki Activity

Tiny LZSA1 ratio increase; small improvements

Browse Source
This commit is contained in:
Emmanuel Marty 2019-12-15 23:36:51 +01:00 committed by GitHub
parent e9ca5032bc
commit a0ac24d105
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 21 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/matchfinder.c
View File

@ -105,7 +105,7 @@ int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *p
} }
} }
else { else {
for (i = 1; i < nInWindowSize - 1; i++) { for (i = 1; i < nInWindowSize; i++) {
int nIndex = (int)(intervals[i] & POS_MASK); int nIndex = (int)(intervals[i] & POS_MASK);
int nLen = PLCP[nIndex]; int nLen = PLCP[nIndex];
if (nLen < nMinMatchSize) if (nLen < nMinMatchSize)
@ -116,9 +116,6 @@ int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *p
} }
} }
if (i < nInWindowSize)
intervals[i] &= POS_MASK;
/** /**
* Build intervals for finding matches * Build intervals for finding matches
* *

14
src/shrink_block_v1.c
View File

@ -172,7 +172,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
for (i = nStartOffset; i != nEndOffset; i++) { for (i = nStartOffset; i != nEndOffset; i++) {
int m; int m;
for (j = 0; j < NMATCHES_PER_ARRIVAL_SMALL && arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].from_slot; j++) { for (j = 0; j < NMATCHES_PER_ARRIVAL_V1 && arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].from_slot; j++) {
int nPrevCost = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].cost; int nPrevCost = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].cost;
int nCodingChoiceCost = nPrevCost + 8 /* literal */; int nCodingChoiceCost = nPrevCost + 8 /* literal */;
int nScore = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].score + 1; int nScore = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].score + 1;
@ -185,7 +185,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
if (!nFavorRatio && nNumLiterals == 1) if (!nFavorRatio && nNumLiterals == 1)
nCodingChoiceCost += MODESWITCH_PENALTY; nCodingChoiceCost += MODESWITCH_PENALTY;
for (n = 0; n < NMATCHES_PER_ARRIVAL_SMALL /* we only need the literals + short match cost + long match cost cases */; n++) { for (n = 0; n < NMATCHES_PER_ARRIVAL_V1 /* we only need the literals + short match cost + long match cost cases */; n++) {
lzsa_arrival *pDestArrival = &arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n]; lzsa_arrival *pDestArrival = &arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n];
if (pDestArrival->from_slot == 0 || if (pDestArrival->from_slot == 0 ||
@ -193,7 +193,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
(nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) { (nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) {
memmove(&arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n + 1], memmove(&arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n + 1],
&arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n], &arrival[((i + 1) << MATCHES_PER_ARRIVAL_SHIFT) + n],
sizeof(lzsa_arrival) * (NMATCHES_PER_ARRIVAL_SMALL - n - 1)); sizeof(lzsa_arrival) * (NMATCHES_PER_ARRIVAL_V1 - n - 1));
pDestArrival->cost = nCodingChoiceCost; pDestArrival->cost = nCodingChoiceCost;
pDestArrival->from_pos = i; pDestArrival->from_pos = i;
@ -225,7 +225,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
for (k = nStartingMatchLen; k <= nMatchLen; k++) { for (k = nStartingMatchLen; k <= nMatchLen; k++) {
int nMatchLenCost = lzsa_get_match_varlen_size_v1(k - MIN_MATCH_SIZE_V1); int nMatchLenCost = lzsa_get_match_varlen_size_v1(k - MIN_MATCH_SIZE_V1);
for (j = 0; j < NMATCHES_PER_ARRIVAL_SMALL && arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].from_slot; j++) { for (j = 0; j < NMATCHES_PER_ARRIVAL_V1 && arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].from_slot; j++) {
int nPrevCost = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].cost; int nPrevCost = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].cost;
int nCodingChoiceCost = nPrevCost + 8 /* token */ /* the actual cost of the literals themselves accumulates up the chain */ + nMatchOffsetCost + nMatchLenCost; int nCodingChoiceCost = nPrevCost + 8 /* token */ /* the actual cost of the literals themselves accumulates up the chain */ + nMatchOffsetCost + nMatchLenCost;
int nScore = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].score + 5; int nScore = arrival[(i << MATCHES_PER_ARRIVAL_SHIFT) + j].score + 5;
@ -235,7 +235,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
nCodingChoiceCost += MODESWITCH_PENALTY; nCodingChoiceCost += MODESWITCH_PENALTY;
for (n = 0; for (n = 0;
n < NMATCHES_PER_ARRIVAL_SMALL && arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].from_slot && arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].cost <= nCodingChoiceCost; n < NMATCHES_PER_ARRIVAL_V1 && arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].from_slot && arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].cost <= nCodingChoiceCost;
n++) { n++) {
if (lzsa_get_offset_cost_v1(arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].rep_offset) == lzsa_get_offset_cost_v1(match[m].offset)) { if (lzsa_get_offset_cost_v1(arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n].rep_offset) == lzsa_get_offset_cost_v1(match[m].offset)) {
exists = 1; exists = 1;
@ -243,7 +243,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
} }
} }
for (n = 0; !exists && n < NMATCHES_PER_ARRIVAL_SMALL /* we only need the literals + short match cost + long match cost cases */; n++) { for (n = 0; !exists && n < NMATCHES_PER_ARRIVAL_V1 /* we only need the literals + short match cost + long match cost cases */; n++) {
lzsa_arrival *pDestArrival = &arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n]; lzsa_arrival *pDestArrival = &arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n];
if (pDestArrival->from_slot == 0 || if (pDestArrival->from_slot == 0 ||
@ -251,7 +251,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *p
(nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) { (nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) {
memmove(&arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n + 1], memmove(&arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n + 1],
&arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n], &arrival[((i + k) << MATCHES_PER_ARRIVAL_SHIFT) + n],
sizeof(lzsa_arrival) * (NMATCHES_PER_ARRIVAL_SMALL - n - 1)); sizeof(lzsa_arrival) * (NMATCHES_PER_ARRIVAL_V1 - n - 1));
pDestArrival->cost = nCodingChoiceCost; pDestArrival->cost = nCodingChoiceCost;
pDestArrival->from_pos = i; pDestArrival->from_pos = i;

13
src/shrink_block_v2.c
View File

@ -183,6 +183,7 @@ static inline int lzsa_write_match_varlen_v2(unsigned char *pOutData, int nOutOf
* @param nMatchOffset match offset to use as rep candidate * @param nMatchOffset match offset to use as rep candidate
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes) * @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
* @param nMatchesPerArrival number of arrivals to record per input buffer position
* @param nDepth current insertion depth * @param nDepth current insertion depth
*/ */
static void lzsa_insert_forward_match_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int i, const int nMatchOffset, const int nStartOffset, const int nEndOffset, const int nMatchesPerArrival, int nDepth) { static void lzsa_insert_forward_match_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int i, const int nMatchOffset, const int nStartOffset, const int nEndOffset, const int nMatchesPerArrival, int nDepth) {
@ -241,15 +242,19 @@ static void lzsa_insert_forward_match_v2(lzsa_compressor *pCompressor, const uns
* *
* @param pCompressor compression context * @param pCompressor compression context
* @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress)
* @param pBestMatch pointer to buffer for outputting optimal matches
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes) * @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
* @param nReduce non-zero to reduce the number of tokens when the path costs are equal, zero not to
* @param nInsertForwardReps non-zero to insert forward repmatch candidates, zero to use the previously inserted candidates * @param nInsertForwardReps non-zero to insert forward repmatch candidates, zero to use the previously inserted candidates
* @param nMatchesPerArrival number of arrivals to record per input buffer position
*/ */
static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset, const int nReduce, const int nInsertForwardReps, const int nMatchesPerArrival) { static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset, const int nReduce, const int nInsertForwardReps, const int nMatchesPerArrival) {
lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << MATCHES_PER_ARRIVAL_SHIFT); lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << MATCHES_PER_ARRIVAL_SHIFT);
const int nFavorRatio = (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) ? 1 : 0; const int nFavorRatio = (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) ? 1 : 0;
const int nMinMatchSize = pCompressor->min_match_size; const int nMinMatchSize = pCompressor->min_match_size;
const int nDisableScore = nReduce ? 0 : (2 * BLOCK_SIZE); const int nDisableScore = nReduce ? 0 : (2 * BLOCK_SIZE);
const int nLeaveAloneMatchSize = (nMatchesPerArrival == NMATCHES_PER_ARRIVAL_V2_SMALL) ? LEAVE_ALONE_MATCH_SIZE_SMALL : LEAVE_ALONE_MATCH_SIZE;
int i, j, n; int i, j, n;
if ((nEndOffset - nStartOffset) > BLOCK_SIZE) return; if ((nEndOffset - nStartOffset) > BLOCK_SIZE) return;
@ -334,7 +339,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
lzsa_match *match = pCompressor->match + ((i - nStartOffset) << MATCHES_PER_INDEX_SHIFT_V2); lzsa_match *match = pCompressor->match + ((i - nStartOffset) << MATCHES_PER_INDEX_SHIFT_V2);
int nMinRepLen[NMATCHES_PER_ARRIVAL_BIG]; int nMinRepLen[NMATCHES_PER_ARRIVAL_V2_BIG];
memset(nMinRepLen, 0, nMatchesPerArrival * sizeof(int)); memset(nMinRepLen, 0, nMatchesPerArrival * sizeof(int));
for (m = 0; m < NMATCHES_PER_INDEX_V2 && match[m].length; m++) { for (m = 0; m < NMATCHES_PER_INDEX_V2 && match[m].length; m++) {
@ -343,7 +348,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
int nScorePenalty = ((match[m].length & 0x8000) >> 15); int nScorePenalty = ((match[m].length & 0x8000) >> 15);
int nNoRepmatchOffsetCost = (nMatchOffset <= 32) ? 4 : ((nMatchOffset <= 512) ? 8 : ((nMatchOffset <= (8192 + 512)) ? 12 : 16)); int nNoRepmatchOffsetCost = (nMatchOffset <= 32) ? 4 : ((nMatchOffset <= 512) ? 8 : ((nMatchOffset <= (8192 + 512)) ? 12 : 16));
int nStartingMatchLen, k; int nStartingMatchLen, k;
int nMaxRepLen[NMATCHES_PER_ARRIVAL_BIG]; int nMaxRepLen[NMATCHES_PER_ARRIVAL_V2_BIG];
if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS)) if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS))
nMatchLen = nEndOffset - LAST_LITERALS - i; nMatchLen = nEndOffset - LAST_LITERALS - i;
@ -375,7 +380,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
lzsa_insert_forward_match_v2(pCompressor, pInWindow, i, nMatchOffset, nStartOffset, nEndOffset, nMatchesPerArrival, 0); lzsa_insert_forward_match_v2(pCompressor, pInWindow, i, nMatchOffset, nStartOffset, nEndOffset, nMatchesPerArrival, 0);
int nMatchLenCost = 0; int nMatchLenCost = 0;
if (nMatchLen >= LEAVE_ALONE_MATCH_SIZE) { if (nMatchLen >= nLeaveAloneMatchSize) {
nStartingMatchLen = nMatchLen; nStartingMatchLen = nMatchLen;
nMatchLenCost = 4 + 24; nMatchLenCost = 4 + 24;
} }
@ -1124,7 +1129,7 @@ static int lzsa_write_raw_uncompressed_block_v2(lzsa_compressor *pCompressor, co
*/ */
int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) { int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) {
int nResult, nBaseCompressedSize; int nResult, nBaseCompressedSize;
int nMatchesPerArrival = (nInDataSize < 65536) ? NMATCHES_PER_ARRIVAL_BIG : NMATCHES_PER_ARRIVAL_SMALL; int nMatchesPerArrival = (nInDataSize < 65536) ? NMATCHES_PER_ARRIVAL_V2_BIG : NMATCHES_PER_ARRIVAL_V2_SMALL;
/* Compress optimally without breaking ties in favor of less tokens */ /* Compress optimally without breaking ties in favor of less tokens */

6
src/shrink_context.h
View File

@ -49,8 +49,9 @@ extern "C" {
#define VISITED_FLAG 0x80000000 #define VISITED_FLAG 0x80000000
#define EXCL_VISITED_MASK 0x7fffffff #define EXCL_VISITED_MASK 0x7fffffff
#define NMATCHES_PER_ARRIVAL_SMALL 8 #define NMATCHES_PER_ARRIVAL_V1 8
#define NMATCHES_PER_ARRIVAL_BIG 32 #define NMATCHES_PER_ARRIVAL_V2_SMALL 9
#define NMATCHES_PER_ARRIVAL_V2_BIG 32
#define MATCHES_PER_ARRIVAL_SHIFT 5 #define MATCHES_PER_ARRIVAL_SHIFT 5
#define NMATCHES_PER_INDEX_V1 8 #define NMATCHES_PER_INDEX_V1 8
@ -60,6 +61,7 @@ extern "C" {
#define MATCHES_PER_INDEX_SHIFT_V2 6 #define MATCHES_PER_INDEX_SHIFT_V2 6
#define LEAVE_ALONE_MATCH_SIZE 300 #define LEAVE_ALONE_MATCH_SIZE 300
#define LEAVE_ALONE_MATCH_SIZE_SMALL 1000
#define LAST_LITERALS 0 #define LAST_LITERALS 0