6502/lzsa
1
0
Fork 0
mirror of https://github.com/emmanuel-marty/lzsa.git synced 2024-11-22 21:32:07 +00:00
Code Issues Projects Releases Wiki Activity

Implement forward arrivals optimal parsers

Browse Source
This commit is contained in:
Emmanuel Marty 2019-08-27 00:51:34 +02:00 committed by GitHub
parent 90b4da64d1
commit ef259e6867
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 327 additions and 327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
src/shrink_block_v1.c
View File

@ -139,7 +139,139 @@ static inline int lzsa_write_match_varlen_v1(unsigned char *pOutData, int nOutOf
} }
/** /**
* Attempt to pick optimal matches, so as to produce the smallest possible output that decompresses to the same input * Get offset encoding cost in bits
*
* @param nMatchOffset offset to get cost of
*
* @return cost in bits
*/
static inline int lzsa_get_offset_cost_v1(const unsigned int nMatchOffset) {
return (nMatchOffset <= 256) ? 8 : 16;
}
/**
* Attempt to pick optimal matches using a forward arrivals parser, so as to produce the smallest possible output that decompresses to the same input
*
* @param pCompressor compression context
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
*/
static void lzsa_optimize_arrivals_v1(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) {
lzsa_arrival *arrival = pCompressor->arrival;
int nMinMatchSize = pCompressor->min_match_size;
int i, j, n;
memset(arrival + (nStartOffset << MATCHES_PER_OFFSET_SHIFT), 0, sizeof(lzsa_arrival) * ((nEndOffset - nStartOffset) << MATCHES_PER_OFFSET_SHIFT));
arrival[nStartOffset << MATCHES_PER_OFFSET_SHIFT].from_slot = -1;
for (i = nStartOffset; i != (nEndOffset - 1); i++) {
const lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT);
int m;
for (j = 0; j < NMATCHES_PER_OFFSET && arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].from_slot; j++) {
int nPrevCost = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].cost;
int nCodingChoiceCost = nPrevCost + 8 /* literal */;
int nNumLiterals = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].num_literals + 1;
if (nNumLiterals == LITERALS_RUN_LEN_V1 || nNumLiterals == 256 || nNumLiterals == 512) {
nCodingChoiceCost += 8;
}
lzsa_arrival *pDestArrival = &arrival[((i + 1) << MATCHES_PER_OFFSET_SHIFT)];
if (pDestArrival->from_slot == 0 ||
nCodingChoiceCost <= pDestArrival->cost) {
memmove(&arrival[((i + 1) << MATCHES_PER_OFFSET_SHIFT) + 1],
&arrival[((i + 1) << MATCHES_PER_OFFSET_SHIFT)],
sizeof(lzsa_arrival) * (NMATCHES_PER_OFFSET - 1));
pDestArrival->cost = nCodingChoiceCost;
pDestArrival->from_pos = i;
pDestArrival->from_slot = j + 1;
pDestArrival->match_offset = 0;
pDestArrival->match_len = 0;
pDestArrival->num_literals = nNumLiterals;
pDestArrival->rep_offset = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].rep_offset;
}
}
for (m = 0; m < NMATCHES_PER_OFFSET && pMatch[m].length >= nMinMatchSize; m++) {
int nMatchLen = pMatch[m].length;
int nMatchOffsetCost = lzsa_get_offset_cost_v1(pMatch[m].offset);
int nStartingMatchLen, k;
if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS))
nMatchLen = nEndOffset - LAST_LITERALS - i;
if (nMatchLen >= LEAVE_ALONE_MATCH_SIZE)
nStartingMatchLen = nMatchLen;
else
nStartingMatchLen = nMinMatchSize;
for (k = nStartingMatchLen; k <= nMatchLen; k++) {
int nMatchLenCost = lzsa_get_match_varlen_size_v1(k - MIN_MATCH_SIZE_V1);
for (j = 0; j < NMATCHES_PER_OFFSET && arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].from_slot; j++) {
int nPrevCost = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].cost;
int nCodingChoiceCost = nPrevCost + 8 /* token */ /* the actual cost of the literals themselves accumulates up the chain */ + nMatchOffsetCost + nMatchLenCost;
for (n = 0; n < 3 /* we only need the literals + short match cost + long match cost cases */; n++) {
lzsa_arrival *pDestArrival = &arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + n];
if (pDestArrival->from_slot == 0 ||
nCodingChoiceCost <= pDestArrival->cost) {
int exists = 0;
for (int l = n; l < NMATCHES_PER_OFFSET && arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + l].from_slot &&
arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + l].cost == nCodingChoiceCost; l++) {
if (lzsa_get_offset_cost_v1(arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + l].rep_offset) == lzsa_get_offset_cost_v1(pMatch[m].offset)) {
exists = 1;
break;
}
}
if (!exists) {
memmove(&arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + n + 1],
&arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + n],
sizeof(lzsa_arrival) * (NMATCHES_PER_OFFSET - n - 1));
pDestArrival->cost = nCodingChoiceCost;
pDestArrival->from_pos = i;
pDestArrival->from_slot = j + 1;
pDestArrival->match_offset = pMatch[m].offset;
pDestArrival->match_len = k;
pDestArrival->num_literals = 0;
pDestArrival->rep_offset = pMatch[m].offset;
}
break;
}
}
}
}
}
}
lzsa_arrival *end_arrival = &arrival[(i << MATCHES_PER_OFFSET_SHIFT) + 0];
pCompressor->match[i << MATCHES_PER_OFFSET_SHIFT].length = 0;
pCompressor->match[i << MATCHES_PER_OFFSET_SHIFT].offset = 0;
int nEndCost = end_arrival->cost;
int *backward_cost = (int*)pCompressor->pos_data; /* Reuse */
for (i = nStartOffset; i != nEndOffset; i++) {
backward_cost[i] = nEndCost - arrival[(i << MATCHES_PER_OFFSET_SHIFT) + 0].cost;
}
while (end_arrival->from_slot > 0 && end_arrival->from_pos >= 0) {
pCompressor->match[end_arrival->from_pos << MATCHES_PER_OFFSET_SHIFT].length = end_arrival->match_len;
pCompressor->match[end_arrival->from_pos << MATCHES_PER_OFFSET_SHIFT].offset = end_arrival->match_offset;
end_arrival = &arrival[(end_arrival->from_pos << MATCHES_PER_OFFSET_SHIFT) + (end_arrival->from_slot - 1)];
}
}
/**
* Attempt to pick optimal matches using a backward LZSS style parser, so as to produce the smallest possible output that decompresses to the same input
* *
* @param pCompressor compression context * @param pCompressor compression context
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes) * @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
@ -504,7 +636,10 @@ static int lzsa_write_raw_uncompressed_block_v1(lzsa_compressor *pCompressor, co
int lzsa_optimize_and_write_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) { int lzsa_optimize_and_write_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) {
int nResult; int nResult;
lzsa_optimize_matches_v1(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); if (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO)
lzsa_optimize_arrivals_v1(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
else
lzsa_optimize_matches_v1(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
int nDidReduce; int nDidReduce;
int nPasses = 0; int nPasses = 0;

435
src/shrink_block_v2.c
View File

@ -178,13 +178,160 @@ static inline int lzsa_write_match_varlen_v2(unsigned char *pOutData, int nOutOf
} }
/** /**
* Attempt to pick optimal matches, so as to produce the smallest possible output that decompresses to the same input * Attempt to pick optimal matches using a forward arrivals parser, so as to produce the smallest possible output that decompresses to the same input
* *
* @param pCompressor compression context * @param pCompressor compression context
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes) * @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
*/ */
static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) { static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) {
lzsa_arrival *arrival = pCompressor->arrival;
int nMinMatchSize = pCompressor->min_match_size;
int i, j, n;
memset(arrival + (nStartOffset << MATCHES_PER_OFFSET_SHIFT), 0, sizeof(lzsa_arrival) * ((nEndOffset - nStartOffset) << MATCHES_PER_OFFSET_SHIFT));
arrival[nStartOffset << MATCHES_PER_OFFSET_SHIFT].from_slot = -1;
for (i = nStartOffset; i != (nEndOffset - 1); i++) {
const lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT);
int m;
for (j = 0; j < NMATCHES_PER_OFFSET && arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].from_slot; j++) {
int nPrevCost = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].cost;
int nCodingChoiceCost = nPrevCost + 8 /* literal */;
int nNumLiterals = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].num_literals + 1;
if (nNumLiterals == LITERALS_RUN_LEN_V2) {
nCodingChoiceCost += 4;
}
else if (nNumLiterals == (LITERALS_RUN_LEN_V2 + 15)) {
nCodingChoiceCost += 8;
}
else if (nNumLiterals == 256) {
nCodingChoiceCost += 16;
}
for (n = 0; n < NMATCHES_PER_OFFSET; n++) {
lzsa_arrival *pDestArrival = &arrival[((i + 1) << MATCHES_PER_OFFSET_SHIFT) + n];
if (pDestArrival->from_slot == 0 ||
nCodingChoiceCost <= pDestArrival->cost) {
int exists = 0;
for (int l = n; l < NMATCHES_PER_OFFSET && arrival[((i + 1) << MATCHES_PER_OFFSET_SHIFT) + l].from_slot &&
arrival[((i + 1) << MATCHES_PER_OFFSET_SHIFT) + l].cost == nCodingChoiceCost; l++) {
if (arrival[((i + 1) << MATCHES_PER_OFFSET_SHIFT) + l].rep_offset == arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].rep_offset) {
exists = 1;
break;
}
}
if (!exists) {
memmove(&arrival[((i + 1) << MATCHES_PER_OFFSET_SHIFT) + n + 1],
&arrival[((i + 1) << MATCHES_PER_OFFSET_SHIFT) + n],
sizeof(lzsa_arrival) * (NMATCHES_PER_OFFSET - n - 1));
pDestArrival->cost = nCodingChoiceCost;
pDestArrival->from_pos = i;
pDestArrival->from_slot = j + 1;
pDestArrival->match_offset = 0;
pDestArrival->match_len = 0;
pDestArrival->num_literals = nNumLiterals;
pDestArrival->rep_offset = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].rep_offset;
}
break;
}
}
}
for (m = 0; m < NMATCHES_PER_OFFSET && pMatch[m].length >= nMinMatchSize; m++) {
int nMatchLen = pMatch[m].length;
int nNoRepMatchOffsetCost = (pMatch[m].offset <= 32) ? 4 : ((pMatch[m].offset <= 512) ? 8 : ((pMatch[m].offset <= (8192 + 512)) ? 12 : 16));
int nStartingMatchLen, k;
if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS))
nMatchLen = nEndOffset - LAST_LITERALS - i;
if (nMatchLen >= LEAVE_ALONE_MATCH_SIZE)
nStartingMatchLen = nMatchLen;
else
nStartingMatchLen = nMinMatchSize;
for (k = nStartingMatchLen; k <= nMatchLen; k++) {
int nMatchLenCost = lzsa_get_match_varlen_size_v2(k - MIN_MATCH_SIZE_V2);
for (j = 0; j < NMATCHES_PER_OFFSET && arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].from_slot; j++) {
int nMatchOffsetCost = (pMatch[m].offset == arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].rep_offset) ? 0 : nNoRepMatchOffsetCost;
int nPrevCost = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].cost;
int nCodingChoiceCost = nPrevCost + 8 /* token */ /* the actual cost of the literals themselves accumulates up the chain */ + nMatchOffsetCost + nMatchLenCost;
for (n = 0; n < NMATCHES_PER_OFFSET; n++) {
lzsa_arrival *pDestArrival = &arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + n];
if (pDestArrival->from_slot == 0 ||
nCodingChoiceCost <= pDestArrival->cost) {
int exists = 0;
for (int l = n; l < NMATCHES_PER_OFFSET && arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + l].from_slot &&
arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + l].cost == nCodingChoiceCost; l++) {
if (arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + l].rep_offset == pMatch[m].offset) {
exists = 1;
break;
}
}
if (!exists) {
memmove(&arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + n + 1],
&arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + n],
sizeof(lzsa_arrival) * (NMATCHES_PER_OFFSET - n - 1));
pDestArrival->cost = nCodingChoiceCost;
pDestArrival->from_pos = i;
pDestArrival->from_slot = j + 1;
pDestArrival->match_offset = pMatch[m].offset;
pDestArrival->match_len = k;
pDestArrival->num_literals = 0;
pDestArrival->rep_offset = pMatch[m].offset;
}
break;
}
}
}
}
}
}
lzsa_arrival *end_arrival = &arrival[(i << MATCHES_PER_OFFSET_SHIFT) + 0];
pCompressor->best_match[i].length = 0;
pCompressor->best_match[i].offset = 0;
unsigned int nRepMatchOffset = 0;
int nEndCost = end_arrival->cost;
int *backward_cost = (int*)pCompressor->pos_data; /* Reuse */
for (i = nStartOffset; i != nEndOffset; i++) {
backward_cost[i] = nEndCost - arrival[(i << MATCHES_PER_OFFSET_SHIFT) + 0].cost;
}
while (end_arrival->from_slot > 0 && end_arrival->from_pos >= 0) {
pCompressor->best_match[end_arrival->from_pos].length = end_arrival->match_len;
pCompressor->best_match[end_arrival->from_pos].offset = end_arrival->match_offset;
pCompressor->repmatch_opt[end_arrival->from_pos].expected_repmatch = (end_arrival->match_len >= MIN_MATCH_SIZE_V2 && nRepMatchOffset == end_arrival->match_offset) ? 1 : 0;
if (end_arrival->match_len >= MIN_MATCH_SIZE_V2)
nRepMatchOffset = end_arrival->match_offset;
end_arrival = &arrival[(end_arrival->from_pos << MATCHES_PER_OFFSET_SHIFT) + (end_arrival->from_slot - 1)];
}
}
/**
* Attempt to pick optimal matches using a backward LZSS style parser, so as to produce the smallest possible output that decompresses to the same input
*
* @param pCompressor compression context
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
*/
static void lzsa_optimize_backward_v2(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) {
int *cost = (int*)pCompressor->pos_data; /* Reuse */ int *cost = (int*)pCompressor->pos_data; /* Reuse */
int *prev_match = (int*)pCompressor->intervals; /* Reuse */ int *prev_match = (int*)pCompressor->intervals; /* Reuse */
lzsa_repmatch_opt *repmatch_opt = pCompressor->repmatch_opt; lzsa_repmatch_opt *repmatch_opt = pCompressor->repmatch_opt;
@ -578,245 +725,6 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, lzsa_mat
return nDidReduce; return nDidReduce;
} }
/**
* Get cost of the best encoding choice at a given offset, going forward
*
* @param pCompressor compression context
* @param i offset in input window
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes)
* @param nNumLiterals current pending number of literals to be encoded with the next token
* @param nRepMatchOffset current rep-match offset
* @param nDepth current recursion depth
* @param pBestMatchLen pointer to returned best match length for the position (0 for no match)
* @param pBestMatchOffset pointer to returned best match offset for the position (if there is a match)
*
* @return cost of best encoding choice for offset
*/
static int lzsa_get_forward_cost_v2(lzsa_compressor *pCompressor, const int i, const int nEndOffset, const int nNumLiterals, const int nRepMatchOffset, int nDepth, int *pBestMatchLen, int *pBestMatchOffset) {
if (i >= nEndOffset)
return 0;
int *cost = (int*)pCompressor->pos_data; /* Reuse */
if (nDepth >= pCompressor->max_forward_depth)
return cost[i];
if (nDepth >= 1) {
unsigned int nValue = 0;
if (!lzsa_hashmap_find(&pCompressor->cost_map, HASH_KEY(nRepMatchOffset, nNumLiterals, i), &nValue))
return nValue;
}
int nMinMatchSize = pCompressor->min_match_size;
int m;
const lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT);
int nBestCost, nBestMatchLen, nBestMatchOffset, nTmpMatchLen, nTmpMatchOffset;
int nLiteralsCost;
nBestCost = 8 + lzsa_get_forward_cost_v2(pCompressor, i + 1, nEndOffset, nNumLiterals + 1, nRepMatchOffset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset);
nBestMatchLen = 0;
nBestMatchOffset = 0;
nLiteralsCost = lzsa_get_literals_varlen_size_v2(nNumLiterals);
for (m = 0; m < NMATCHES_PER_OFFSET && pMatch[m].length >= nMinMatchSize; m++) {
if (pMatch[m].length > 30) {
int nCurCost;
int nMatchLen = pMatch[m].length;
if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS))
nMatchLen = nEndOffset - LAST_LITERALS - i;
int nMatchOffsetSize;
if (nRepMatchOffset == pMatch[m].offset)
nMatchOffsetSize = 0;
else {
nMatchOffsetSize = (pMatch[m].offset <= 32) ? 4 : ((pMatch[m].offset <= 512) ? 8 : ((pMatch[m].offset <= (8192 + 512)) ? 12 : 16));
}
nCurCost = 8 + nLiteralsCost + nMatchOffsetSize + lzsa_get_match_varlen_size_v2(nMatchLen - MIN_MATCH_SIZE_V2);
nCurCost += lzsa_get_forward_cost_v2(pCompressor, i + nMatchLen, nEndOffset, 0, pMatch[m].offset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset);
if (nBestCost >= nCurCost) {
nBestCost = nCurCost;
nBestMatchLen = nMatchLen;
nBestMatchOffset = pMatch[m].offset;
}
}
else {
int nMatchLen = pMatch[m].length;
int k, nMatchRunLen;
if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS))
nMatchLen = nEndOffset - LAST_LITERALS - i;
nMatchRunLen = nMatchLen;
if (nMatchRunLen > MATCH_RUN_LEN_V2)
nMatchRunLen = MATCH_RUN_LEN_V2;
for (k = nMinMatchSize; k < nMatchRunLen; k++) {
int nCurCost;
int nMatchOffsetSize;
if (nRepMatchOffset == pMatch[m].offset)
nMatchOffsetSize = 0;
else {
nMatchOffsetSize = (pMatch[m].offset <= 32) ? 4 : ((pMatch[m].offset <= 512) ? 8 : ((pMatch[m].offset <= (8192 + 512)) ? 12 : 16));
}
nCurCost = 8 + nLiteralsCost + nMatchOffsetSize /* no extra match len bytes */;
nCurCost += lzsa_get_forward_cost_v2(pCompressor, i + k, nEndOffset, 0, pMatch[m].offset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset);
if (nBestCost >= nCurCost) {
nBestCost = nCurCost;
nBestMatchLen = k;
nBestMatchOffset = pMatch[m].offset;
}
}
for (; k <= nMatchLen; k++) {
int nCurCost;
int nMatchOffsetSize;
if (nRepMatchOffset == pMatch[m].offset)
nMatchOffsetSize = 0;
else {
nMatchOffsetSize = (pMatch[m].offset <= 32) ? 4 : ((pMatch[m].offset <= 512) ? 8 : ((pMatch[m].offset <= (8192 + 512)) ? 12 : 16));
}
nCurCost = 8 + nLiteralsCost + nMatchOffsetSize + lzsa_get_match_varlen_size_v2(k - MIN_MATCH_SIZE_V2);
nCurCost += lzsa_get_forward_cost_v2(pCompressor, i + k, nEndOffset, 0, pMatch[m].offset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset);
if (nBestCost >= nCurCost) {
nBestCost = nCurCost;
nBestMatchLen = k;
nBestMatchOffset = pMatch[m].offset;
}
}
}
}
*pBestMatchLen = nBestMatchLen;
*pBestMatchOffset = nBestMatchOffset;
lzsa_hashmap_insert(&pCompressor->cost_map, HASH_KEY(nRepMatchOffset, nNumLiterals, i), nBestCost);
return nBestCost;
}
/**
* Attempt to further improve the selected optimal matches with a chain-N forward parser pass
*
* @param pCompressor compression context
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
*/
static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) {
int i;
int nNumLiterals = 0;
int nRepMatchOffset = 0;
int *cost = (int*)pCompressor->pos_data; /* Reuse */
lzsa_hashmap_init(&pCompressor->cost_map);
for (i = nStartOffset; i < nEndOffset; ) {
int nBestMatchLen = 0, nBestMatchOffset = 0;
int nBestCost = lzsa_get_forward_cost_v2(pCompressor, i, nEndOffset, nNumLiterals, nRepMatchOffset, 0, &nBestMatchLen, &nBestMatchOffset);
lzsa_hashmap_clear(&pCompressor->cost_map);
lzsa_match *pMatch = pCompressor->improved_match + i;
if (nBestCost < cost[i]) {
pMatch->length = nBestMatchLen;
pMatch->offset = nBestMatchOffset;
}
if (pMatch->length >= MIN_MATCH_SIZE_V2) {
nNumLiterals = 0;
nRepMatchOffset = pMatch->offset;
i += pMatch->length;
}
else {
nNumLiterals++;
i++;
}
}
lzsa_hashmap_clear(&pCompressor->cost_map);
}
/**
* Calculate compressed size
*
* @param pCompressor compression context
* @param pBestMatch optimal matches to evaluate
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
*
* @return compressed size, in bits
*/
static int lzsa_get_compressed_size_v2(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) {
int i;
int nNumLiterals = 0;
int nCompressedSize = 0;
int nRepMatchOffset = 0;
for (i = nStartOffset; i < nEndOffset; ) {
const lzsa_match *pMatch = pBestMatch + i;
if (pMatch->length >= MIN_MATCH_SIZE_V2) {
int nMatchOffset = pMatch->offset;
int nMatchLen = pMatch->length;
int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE_V2;
int nOffsetSize;
if (nMatchOffset == nRepMatchOffset) {
nOffsetSize = 0;
}
else {
if (nMatchOffset <= 32) {
nOffsetSize = 4;
}
else if (nMatchOffset <= 512) {
nOffsetSize = 8;
}
else if (nMatchOffset <= (8192 + 512)) {
nOffsetSize = 12;
}
else {
nOffsetSize = 16;
}
}
int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3) + nOffsetSize /* match offset */ + lzsa_get_match_varlen_size_v2(nEncodedMatchLen);
nCompressedSize += nCommandSize;
nNumLiterals = 0;
nRepMatchOffset = nMatchOffset;
i += nMatchLen;
}
else {
nNumLiterals++;
i++;
}
}
{
int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3);
nCompressedSize += nCommandSize;
nNumLiterals = 0;
}
if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
nCompressedSize += 8 + 4 + 8;
}
return nCompressedSize;
}
/** /**
* Emit block of compressed data * Emit block of compressed data
* *
@ -1055,44 +963,19 @@ static int lzsa_write_raw_uncompressed_block_v2(lzsa_compressor *pCompressor, co
int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) { int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) {
int nResult; int nResult;
lzsa_optimize_matches_v2(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); if (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO)
lzsa_match *pBestMatch;
if (pCompressor->max_forward_depth > 0) {
memcpy(pCompressor->improved_match, pCompressor->best_match, nInDataSize * sizeof(lzsa_match));
lzsa_optimize_forward_v2(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); lzsa_optimize_forward_v2(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
else
lzsa_optimize_backward_v2(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
int nDidReduce; int nDidReduce;
int nPasses = 0; int nPasses = 0;
do { do {
nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
nPasses++; nPasses++;
} while (nDidReduce && nPasses < 20); } while (nDidReduce && nPasses < 20);
nPasses = 0; nResult = lzsa_write_block_v2(pCompressor, pCompressor->best_match, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
do {
nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pCompressor->improved_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
nPasses++;
} while (nDidReduce && nPasses < 20);
int nBestCost = lzsa_get_compressed_size_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
int nImprovedCost = lzsa_get_compressed_size_v2(pCompressor, pCompressor->improved_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
if (nBestCost > nImprovedCost)
pBestMatch = pCompressor->improved_match;
else
pBestMatch = pCompressor->best_match;
}
else {
int nDidReduce;
int nPasses = 0;
do {
nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
nPasses++;
} while (nDidReduce && nPasses < 20);
pBestMatch = pCompressor->best_match;
}
nResult = lzsa_write_block_v2(pCompressor, pBestMatch, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
nResult = lzsa_write_raw_uncompressed_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); nResult = lzsa_write_raw_uncompressed_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
} }

34
src/shrink_context.c
View File

@ -61,15 +61,14 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
pCompressor->match = NULL; pCompressor->match = NULL;
pCompressor->selected_match = NULL; pCompressor->selected_match = NULL;
pCompressor->best_match = NULL; pCompressor->best_match = NULL;
pCompressor->improved_match = NULL;
pCompressor->slot_cost = NULL; pCompressor->slot_cost = NULL;
pCompressor->repmatch_opt = NULL; pCompressor->repmatch_opt = NULL;
pCompressor->arrival = NULL;
pCompressor->min_match_size = nMinMatchSize; pCompressor->min_match_size = nMinMatchSize;
if (pCompressor->min_match_size < nMinMatchSizeForFormat) if (pCompressor->min_match_size < nMinMatchSizeForFormat)
pCompressor->min_match_size = nMinMatchSizeForFormat; pCompressor->min_match_size = nMinMatchSizeForFormat;
else if (pCompressor->min_match_size > nMaxMinMatchForFormat) else if (pCompressor->min_match_size > nMaxMinMatchForFormat)
pCompressor->min_match_size = nMaxMinMatchForFormat; pCompressor->min_match_size = nMaxMinMatchForFormat;
pCompressor->max_forward_depth = 0;
pCompressor->format_version = nFormatVersion; pCompressor->format_version = nFormatVersion;
pCompressor->flags = nFlags; pCompressor->flags = nFlags;
pCompressor->safe_dist = 0; pCompressor->safe_dist = 0;
@ -88,16 +87,17 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
pCompressor->match = (lzsa_match *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(lzsa_match)); pCompressor->match = (lzsa_match *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(lzsa_match));
if (pCompressor->match) { if (pCompressor->match) {
if (pCompressor->format_version == 2) { if (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO)
pCompressor->selected_match = (lzsa_match *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(lzsa_match)); pCompressor->arrival = (lzsa_arrival *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(lzsa_arrival));
if (pCompressor->selected_match) { if (pCompressor->arrival || (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) == 0) {
pCompressor->best_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match)); if (pCompressor->format_version == 2) {
pCompressor->selected_match = (lzsa_match *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(lzsa_match));
if (pCompressor->best_match) { if (pCompressor->selected_match) {
pCompressor->improved_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match)); pCompressor->best_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match));
if (pCompressor->improved_match) { if (pCompressor->best_match) {
pCompressor->slot_cost = (int *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(int)); pCompressor->slot_cost = (int *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(int));
if (pCompressor->slot_cost) { if (pCompressor->slot_cost) {
@ -109,9 +109,9 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
} }
} }
} }
} else {
else { return 0;
return 0; }
} }
} }
} }
@ -131,6 +131,11 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
void lzsa_compressor_destroy(lzsa_compressor *pCompressor) { void lzsa_compressor_destroy(lzsa_compressor *pCompressor) {
divsufsort_destroy(&pCompressor->divsufsort_context); divsufsort_destroy(&pCompressor->divsufsort_context);
if (pCompressor->arrival) {
free(pCompressor->arrival);
pCompressor->arrival = NULL;
}
if (pCompressor->repmatch_opt) { if (pCompressor->repmatch_opt) {
free(pCompressor->repmatch_opt); free(pCompressor->repmatch_opt);
pCompressor->repmatch_opt = NULL; pCompressor->repmatch_opt = NULL;
@ -141,11 +146,6 @@ void lzsa_compressor_destroy(lzsa_compressor *pCompressor) {
pCompressor->slot_cost = NULL; pCompressor->slot_cost = NULL;
} }
if (pCompressor->improved_match) {
free(pCompressor->improved_match);
pCompressor->improved_match = NULL;
}
if (pCompressor->best_match) { if (pCompressor->best_match) {
free(pCompressor->best_match); free(pCompressor->best_match);
pCompressor->best_match = NULL; pCompressor->best_match = NULL;

16
src/shrink_context.h
View File

@ -71,6 +71,19 @@ typedef struct _lzsa_repmatch_opt {
short expected_repmatch; short expected_repmatch;
} lzsa_repmatch_opt; } lzsa_repmatch_opt;
/** Forward arrival slot */
typedef struct {
int cost;
int from_pos;
short from_slot;
unsigned short rep_offset;
int num_literals;
unsigned short match_offset;
unsigned short match_len;
} lzsa_arrival;
/** Compression context */ /** Compression context */
typedef struct _lzsa_compressor { typedef struct _lzsa_compressor {
divsufsort_ctx_t divsufsort_context; divsufsort_ctx_t divsufsort_context;
@ -80,11 +93,10 @@ typedef struct _lzsa_compressor {
lzsa_match *match; lzsa_match *match;
lzsa_match *selected_match; lzsa_match *selected_match;
lzsa_match *best_match; lzsa_match *best_match;
lzsa_match *improved_match;
int *slot_cost; int *slot_cost;
lzsa_repmatch_opt *repmatch_opt; lzsa_repmatch_opt *repmatch_opt;
lzsa_arrival *arrival;
int min_match_size; int min_match_size;
int max_forward_depth;
int format_version; int format_version;
int flags; int flags;
int safe_dist; int safe_dist;

15
src/shrink_inmem.c
View File

@ -84,21 +84,6 @@ size_t lzsa_compress_inmem(unsigned char *pInputData, unsigned char *pOutBuffer,
} }
} }
if ((compressor.flags & LZSA_FLAG_FAVOR_RATIO)) {
if (nInputSize < 16384)
compressor.max_forward_depth = 25;
else {
if (nInputSize < 32768)
compressor.max_forward_depth = 15;
else {
if (nInputSize < BLOCK_SIZE)
compressor.max_forward_depth = 10;
else
compressor.max_forward_depth = 0;
}
}
}
int nPreviousBlockSize = 0; int nPreviousBlockSize = 0;
int nNumBlocks = 0; int nNumBlocks = 0;

15
src/shrink_streaming.c
View File

@ -202,21 +202,6 @@ lzsa_status_t lzsa_compress_stream(lzsa_stream_t *pInStream, lzsa_stream_t *pOut
} }
nDictionaryDataSize = 0; nDictionaryDataSize = 0;
if (nNumBlocks == 0 && (compressor.flags & LZSA_FLAG_FAVOR_RATIO)) {
if (nInDataSize < 16384)
compressor.max_forward_depth = 25;
else {
if (nInDataSize < 32768)
compressor.max_forward_depth = 15;
else {
if (nInDataSize < BLOCK_SIZE)
compressor.max_forward_depth = 10;
else
compressor.max_forward_depth = 0;
}
}
}
int nOutDataSize; int nOutDataSize;
nOutDataSize = lzsa_compressor_shrink_block(&compressor, pInData + BLOCK_SIZE - nPreviousBlockSize, nPreviousBlockSize, nInDataSize, pOutData, ((nInDataSize + nRawPadding) >= BLOCK_SIZE) ? BLOCK_SIZE : (nInDataSize + nRawPadding)); nOutDataSize = lzsa_compressor_shrink_block(&compressor, pInData + BLOCK_SIZE - nPreviousBlockSize, nPreviousBlockSize, nInDataSize, pOutData, ((nInDataSize + nRawPadding) >= BLOCK_SIZE) ? BLOCK_SIZE : (nInDataSize + nRawPadding));