6502/lzsa
1
0
Fork 0
mirror of https://github.com/emmanuel-marty/lzsa.git synced 2025-03-02 19:29:14 +00:00
Code Issues Projects Releases Wiki Activity

Newly compressed LZSA2 files depack 0.7% faster

Browse Source
This commit is contained in:
Emmanuel Marty 2019-10-11 09:05:58 +02:00 committed by GitHub
parent b5117c3dfe
commit baa53f6889
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 159 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
src/shrink_block_v2.c
View File

@ -183,12 +183,12 @@ static inline int lzsa_write_match_varlen_v2(unsigned char *pOutData, int nOutOf
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
*/
static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset) {
static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset, const int nReduce) {
lzsa_arrival *arrival = pCompressor->arrival;
const int nFavorRatio = (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) ? 1 : 0;
const int nMinMatchSize = pCompressor->min_match_size;
const int nDisableScore = nReduce ? 0 : (2 * BLOCK_SIZE);
int i, j, n;
lzsa_match match[32];
memset(arrival + (nStartOffset << MATCHES_PER_OFFSET_SHIFT), 0, sizeof(lzsa_arrival) * ((nEndOffset - nStartOffset) << MATCHES_PER_OFFSET_SHIFT));
@ -199,7 +199,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
arrival[nStartOffset << MATCHES_PER_OFFSET_SHIFT].from_slot = -1;
for (i = nStartOffset; i != (nEndOffset - 1); i++) {
int m, nMatches;
int m;
for (j = 0; j < NMATCHES_PER_OFFSET && arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].from_slot; j++) {
const int nPrevCost = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].cost & 0x3fffffff;
@ -232,9 +232,11 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
}
if (!exists) {
int nScore = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].score + 1;
for (n = 0; n < NMATCHES_PER_OFFSET; n++) {
lzsa_arrival *pDestArrival = &pDestSlots[n];
if (nCodingChoiceCost <= pDestArrival->cost) {
if (nCodingChoiceCost < pDestArrival->cost ||
(nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) {
if (pDestArrival->from_slot) {
memmove(&pDestSlots[n + 1],
@ -248,6 +250,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
pDestArrival->match_offset = 0;
pDestArrival->match_len = 0;
pDestArrival->num_literals = nNumLiterals;
pDestArrival->score = nScore;
pDestArrival->rep_offset = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].rep_offset;
break;
}
@ -256,9 +259,9 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
}
}
nMatches = lzsa_find_matches_at(pCompressor, i, match, 32);
lzsa_match *match = pCompressor->match + (i << 5);
for (m = 0; m < nMatches; m++) {
for (m = 0; m < 32 && match[m].length; m++) {
int nMatchLen = match[m].length;
int nMatchOffset = match[m].offset;
int nNoRepmatchOffsetCost = (nMatchOffset <= 32) ? 4 : ((nMatchOffset <= 512) ? 8 : ((nMatchOffset <= (8192 + 512)) ? 12 : 16));
@ -319,10 +322,13 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
}
if (!exists) {
int nScore = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].score + ((nMatchOffset == nRepOffset) ? 2 : 3);
for (n = 0; n < NMATCHES_PER_OFFSET; n++) {
lzsa_arrival *pDestArrival = &pDestSlots[n];
if (nCodingChoiceCost <= pDestArrival->cost) {
if (nCodingChoiceCost < pDestArrival->cost ||
(nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) {
if (pDestArrival->from_slot) {
memmove(&pDestSlots[n + 1],
&pDestSlots[n],
@ -335,6 +341,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
pDestArrival->match_offset = nMatchOffset;
pDestArrival->match_len = k;
pDestArrival->num_literals = 0;
pDestArrival->score = nScore;
pDestArrival->rep_offset = nMatchOffset;
break;
}
@ -361,10 +368,13 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
}
if (!exists) {
int nScore = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].score + 2;
for (n = 0; n < NMATCHES_PER_OFFSET; n++) {
lzsa_arrival *pDestArrival = &pDestSlots[n];
if (nRepCodingChoiceCost <= pDestArrival->cost) {
if (nRepCodingChoiceCost < pDestArrival->cost ||
(nRepCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) {
if (pDestArrival->from_slot) {
memmove(&pDestSlots[n + 1],
&pDestSlots[n],
@ -377,6 +387,7 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
pDestArrival->match_offset = nRepOffset;
pDestArrival->match_len = k;
pDestArrival->num_literals = 0;
pDestArrival->score = nScore;
pDestArrival->rep_offset = nRepOffset;
break;
}
@ -390,12 +401,12 @@ static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigne
}
lzsa_arrival *end_arrival = &arrival[(i << MATCHES_PER_OFFSET_SHIFT) + 0];
pCompressor->best_match[i].length = 0;
pCompressor->best_match[i].offset = 0;
pBestMatch[i].length = 0;
pBestMatch[i].offset = 0;
while (end_arrival->from_slot > 0 && end_arrival->from_pos >= 0) {
pCompressor->best_match[end_arrival->from_pos].length = end_arrival->match_len;
pCompressor->best_match[end_arrival->from_pos].offset = end_arrival->match_offset;
pBestMatch[end_arrival->from_pos].length = end_arrival->match_len;
pBestMatch[end_arrival->from_pos].offset = end_arrival->match_offset;
end_arrival = &arrival[(end_arrival->from_pos << MATCHES_PER_OFFSET_SHIFT) + (end_arrival->from_slot - 1)];
}
}
@ -568,6 +579,78 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const un
return nDidReduce;
}
/**
* Get compressed data block size
*
* @param pCompressor compression context
* @param pBestMatch optimal matches to emit
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
*
* @return size of compressed data that will be written to output buffer
*/
static int lzsa_get_compressed_size_v2(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) {
int i;
int nNumLiterals = 0;
int nOutOffset = 0;
int nRepMatchOffset = 0;
int nCompressedSize = 0;
for (i = nStartOffset; i < nEndOffset; ) {
const lzsa_match *pMatch = pBestMatch + i;
if (pMatch->length >= MIN_MATCH_SIZE_V2) {
int nMatchOffset = pMatch->offset;
int nMatchLen = pMatch->length;
int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE_V2;
int nOffsetSize;
if (nMatchOffset == nRepMatchOffset) {
nOffsetSize = 0;
}
else {
if (nMatchOffset <= 32) {
nOffsetSize = 4;
}
else if (nMatchOffset <= 512) {
nOffsetSize = 8;
}
else if (nMatchOffset <= (8192 + 512)) {
nOffsetSize = 12;
}
else {
nOffsetSize = 16;
}
}
int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3) + nOffsetSize /* match offset */ + lzsa_get_match_varlen_size_v2(nEncodedMatchLen);
nCompressedSize += nCommandSize;
nNumLiterals = 0;
nRepMatchOffset = nMatchOffset;
i += nMatchLen;
}
else {
nNumLiterals++;
i++;
}
}
{
int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V2) ? LITERALS_RUN_LEN_V2 : nNumLiterals;
int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3);
nCompressedSize += nCommandSize;
nNumLiterals = 0;
}
if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
nCompressedSize += (8 + 4 + 8);
}
return nCompressedSize;
}
/**
* Emit block of compressed data
*
@ -852,9 +935,21 @@ static int lzsa_write_raw_uncompressed_block_v2(lzsa_compressor *pCompressor, co
* @return size of compressed data in output buffer, or -1 if the data is uncompressible
*/
int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) {
int nResult;
int i, nResult, nBaseCompressedSize;
lzsa_optimize_forward_v2(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
/* Find matches */
for (i = nPreviousBlockSize; i != (nPreviousBlockSize + nInDataSize - 1); i++) {
int nMatches = lzsa_find_matches_at(pCompressor, i, pCompressor->match + (i << 5), 32);
while (nMatches < 32) {
pCompressor->match[(i << 5) + nMatches].offset = 0;
pCompressor->match[(i << 5) + nMatches].length = 0;
nMatches++;
}
}
/* Compress optimally without breaking ties in favor of less tokens */
lzsa_optimize_forward_v2(pCompressor, pInWindow, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 0 /* reduce */);
int nDidReduce;
int nPasses = 0;
@ -863,7 +958,29 @@ int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigne
nPasses++;
} while (nDidReduce && nPasses < 20);
nResult = lzsa_write_block_v2(pCompressor, pCompressor->best_match, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
nBaseCompressedSize = lzsa_get_compressed_size_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
lzsa_match *pBestMatch = pCompressor->best_match;
if (nBaseCompressedSize > 0 && nInDataSize < 65536) {
int nReducedCompressedSize;
/* Compress optimally and do break ties in favor of less tokens */
lzsa_optimize_forward_v2(pCompressor, pInWindow, pCompressor->improved_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 1 /* reduce */);
nPasses = 0;
do {
nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pInWindow, pCompressor->improved_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
nPasses++;
} while (nDidReduce && nPasses < 20);
nReducedCompressedSize = lzsa_get_compressed_size_v2(pCompressor, pCompressor->improved_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
if (nReducedCompressedSize > 0 && nReducedCompressedSize <= nBaseCompressedSize) {
/* Pick the parse with the reduced number of tokens as it didn't negatively affect the size */
pBestMatch = pCompressor->improved_match;
}
}
nResult = lzsa_write_block_v2(pCompressor, pBestMatch, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
nResult = lzsa_write_raw_uncompressed_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
}

25
src/shrink_context.c
View File

@ -58,7 +58,9 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
pCompressor->intervals = NULL;
pCompressor->pos_data = NULL;
pCompressor->open_intervals = NULL;
pCompressor->match = NULL;
pCompressor->best_match = NULL;
pCompressor->improved_match = NULL;
pCompressor->arrival = NULL;
pCompressor->min_match_size = nMinMatchSize;
if (pCompressor->min_match_size < nMinMatchSizeForFormat)
@ -93,7 +95,18 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
pCompressor->best_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match));
if (pCompressor->best_match) {
return 0;
if (pCompressor->format_version == 2) {
pCompressor->improved_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match));
if (pCompressor->improved_match) {
pCompressor->match = (lzsa_match *)malloc(nMaxWindowSize * 32 * sizeof(lzsa_match));
if (pCompressor->match)
return 0;
}
}
else {
return 0;
}
}
}
}
@ -113,6 +126,16 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
void lzsa_compressor_destroy(lzsa_compressor *pCompressor) {
divsufsort_destroy(&pCompressor->divsufsort_context);
if (pCompressor->match) {
free(pCompressor->match);
pCompressor->match = NULL;
}
if (pCompressor->improved_match) {
free(pCompressor->improved_match);
pCompressor->improved_match = NULL;
}
if (pCompressor->arrival) {
free(pCompressor->arrival);
pCompressor->arrival = NULL;

3
src/shrink_context.h
View File

@ -74,6 +74,7 @@ typedef struct {
unsigned short rep_offset;
int num_literals;
int score;
unsigned short match_offset;
unsigned short match_len;
@ -114,7 +115,9 @@ typedef struct _lzsa_compressor {
unsigned int *intervals;
unsigned int *pos_data;
unsigned int *open_intervals;
lzsa_match *match;
lzsa_match *best_match;
lzsa_match *improved_match;
lzsa_arrival *arrival;
int min_match_size;
int format_version;