6502/lzsa
1
0
Fork 0
mirror of https://github.com/emmanuel-marty/lzsa.git synced 2025-04-11 23:37:04 +00:00
Code Issues Projects Releases Wiki Activity

Reduce memory used for compression

Browse Source
This commit is contained in:
Emmanuel Marty 2019-09-24 00:21:17 +02:00 committed by GitHub
parent 249b8a4c46
commit de0ff5d3b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 32 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
src/matchfinder.c
View File

@ -283,37 +283,3 @@ void lzsa_skip_matches(lzsa_compressor *pCompressor, const int nStartOffset, con
lzsa_find_matches_at(pCompressor, i, &match, 0);
}
}
/**
* Find all matches for the data to be compressed. Up to NMATCHES_PER_OFFSET matches are stored for each offset, for
* the optimizer to look at.
*
* @param pCompressor compression context
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
*/
void lzsa_find_all_matches(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) {
lzsa_match *pMatch = pCompressor->match + (nStartOffset << MATCHES_PER_OFFSET_SHIFT);
int i;
for (i = nStartOffset; i < nEndOffset; i++) {
int nMatches = lzsa_find_matches_at(pCompressor, i, pMatch, NMATCHES_PER_OFFSET);
int m;
for (m = 0; m < NMATCHES_PER_OFFSET; m++) {
if (nMatches <= m || i > (nEndOffset - LAST_MATCH_OFFSET)) {
pMatch->length = 0;
pMatch->offset = 0;
}
else {
int nMaxLen = (nEndOffset - LAST_LITERALS) - i;
if (nMaxLen < 0)
nMaxLen = 0;
if (pMatch->length > nMaxLen)
pMatch->length = (unsigned short)nMaxLen;
}
pMatch++;
}
}
}

10
src/matchfinder.h
View File

@ -73,16 +73,6 @@ int lzsa_find_matches_at(lzsa_compressor *pCompressor, const int nOffset, lzsa_m
*/
void lzsa_skip_matches(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset);
/**
* Find all matches for the data to be compressed. Up to NMATCHES_PER_OFFSET matches are stored for each offset, for
* the optimizer to look at.
*
* @param pCompressor compression context
* @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
* @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
*/
void lzsa_find_all_matches(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset);
#ifdef __cplusplus
}
#endif

45
src/shrink_block_v1.c
View File

@ -35,6 +35,7 @@
#include "lib.h"
#include "shrink_block_v1.h"
#include "format.h"
#include "matchfinder.h"
/**
* Get the number of extra bits required to represent a literals length
@ -161,14 +162,14 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, const int nSt
const int nMinMatchSize = pCompressor->min_match_size;
const int nFavorRatio = (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) ? 1 : 0;
int i, j, n;
lzsa_match match[8];
memset(arrival + (nStartOffset << MATCHES_PER_OFFSET_SHIFT), 0, sizeof(lzsa_arrival) * ((nEndOffset - nStartOffset) << MATCHES_PER_OFFSET_SHIFT));
arrival[nStartOffset << MATCHES_PER_OFFSET_SHIFT].from_slot = -1;
for (i = nStartOffset; i != (nEndOffset - 1); i++) {
const lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT);
int m;
int m, nMatches;
for (j = 0; j < NMATCHES_PER_OFFSET && arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].from_slot; j++) {
int nPrevCost = arrival[(i << MATCHES_PER_OFFSET_SHIFT) + j].cost;
@ -200,9 +201,11 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, const int nSt
}
}
for (m = 0; m < NMATCHES_PER_OFFSET && pMatch[m].length >= nMinMatchSize; m++) {
int nMatchLen = pMatch[m].length;
int nMatchOffsetCost = lzsa_get_offset_cost_v1(pMatch[m].offset);
nMatches = lzsa_find_matches_at(pCompressor, i, match, 8);
for (m = 0; m < nMatches; m++) {
int nMatchLen = match[m].length;
int nMatchOffsetCost = lzsa_get_offset_cost_v1(match[m].offset);
int nStartingMatchLen, k;
if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS))
@ -226,7 +229,7 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, const int nSt
for (n = 0;
n < 3 && arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + n].from_slot && arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + n].cost <= nCodingChoiceCost;
n++) {
if (lzsa_get_offset_cost_v1(arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + n].rep_offset) == lzsa_get_offset_cost_v1(pMatch[m].offset)) {
if (lzsa_get_offset_cost_v1(arrival[((i + k) << MATCHES_PER_OFFSET_SHIFT) + n].rep_offset) == lzsa_get_offset_cost_v1(match[m].offset)) {
exists = 1;
break;
}
@ -244,10 +247,10 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, const int nSt
pDestArrival->cost = nCodingChoiceCost;
pDestArrival->from_pos = i;
pDestArrival->from_slot = j + 1;
pDestArrival->match_offset = pMatch[m].offset;
pDestArrival->match_offset = match[m].offset;
pDestArrival->match_len = k;
pDestArrival->num_literals = 0;
pDestArrival->rep_offset = pMatch[m].offset;
pDestArrival->rep_offset = match[m].offset;
break;
}
}
@ -257,12 +260,12 @@ static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, const int nSt
}
lzsa_arrival *end_arrival = &arrival[(i << MATCHES_PER_OFFSET_SHIFT) + 0];
pCompressor->match[i << MATCHES_PER_OFFSET_SHIFT].length = 0;
pCompressor->match[i << MATCHES_PER_OFFSET_SHIFT].offset = 0;
pCompressor->best_match[i].length = 0;
pCompressor->best_match[i].offset = 0;
while (end_arrival->from_slot > 0 && end_arrival->from_pos >= 0) {
pCompressor->match[end_arrival->from_pos << MATCHES_PER_OFFSET_SHIFT].length = end_arrival->match_len;
pCompressor->match[end_arrival->from_pos << MATCHES_PER_OFFSET_SHIFT].offset = end_arrival->match_offset;
pCompressor->best_match[end_arrival->from_pos].length = end_arrival->match_len;
pCompressor->best_match[end_arrival->from_pos].offset = end_arrival->match_offset;
end_arrival = &arrival[(end_arrival->from_pos << MATCHES_PER_OFFSET_SHIFT) + (end_arrival->from_slot - 1)];
}
@ -284,7 +287,7 @@ static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const in
int nDidReduce = 0;
for (i = nStartOffset; i < nEndOffset; ) {
lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT);
lzsa_match *pMatch = pCompressor->best_match + i;
if (pMatch->length >= MIN_MATCH_SIZE_V1) {
if (pMatch->length <= 9 /* Don't waste time considering large matches, they will always win over literals */ &&
@ -292,7 +295,7 @@ static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const in
int nNextIndex = i + pMatch->length;
int nNextLiterals = 0;
while (nNextIndex < nEndOffset && pCompressor->match[nNextIndex << MATCHES_PER_OFFSET_SHIFT].length < MIN_MATCH_SIZE_V1) {
while (nNextIndex < nEndOffset && pCompressor->best_match[nNextIndex].length < MIN_MATCH_SIZE_V1) {
nNextLiterals++;
nNextIndex++;
}
@ -306,7 +309,7 @@ static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const in
int j;
for (j = 0; j < nMatchLen; j++) {
pCompressor->match[(i + j) << MATCHES_PER_OFFSET_SHIFT].length = 0;
pCompressor->best_match[i + j].length = 0;
}
nDidReduce = 1;
@ -315,15 +318,15 @@ static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const in
}
if ((i + pMatch->length) < nEndOffset && pMatch->length >= LCP_MAX &&
pMatch->offset && pMatch->offset <= 32 && pCompressor->match[(i + pMatch->length) << MATCHES_PER_OFFSET_SHIFT].offset == pMatch->offset && (pMatch->length % pMatch->offset) == 0 &&
(pMatch->length + pCompressor->match[(i + pMatch->length) << MATCHES_PER_OFFSET_SHIFT].length) <= MAX_VARLEN) {
pMatch->offset && pMatch->offset <= 32 && pCompressor->best_match[i + pMatch->length].offset == pMatch->offset && (pMatch->length % pMatch->offset) == 0 &&
(pMatch->length + pCompressor->best_match[i + pMatch->length].length) <= MAX_VARLEN) {
int nMatchLen = pMatch->length;
/* Join */
pMatch->length += pCompressor->match[(i + nMatchLen) << MATCHES_PER_OFFSET_SHIFT].length;
pCompressor->match[(i + nMatchLen) << MATCHES_PER_OFFSET_SHIFT].offset = 0;
pCompressor->match[(i + nMatchLen) << MATCHES_PER_OFFSET_SHIFT].length = -1;
pMatch->length += pCompressor->best_match[i + nMatchLen].length;
pCompressor->best_match[i + nMatchLen].offset = 0;
pCompressor->best_match[i + nMatchLen].length = -1;
continue;
}
@ -358,7 +361,7 @@ static int lzsa_write_block_v1(lzsa_compressor *pCompressor, const unsigned char
int nOutOffset = 0;
for (i = nStartOffset; i < nEndOffset; ) {
lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT);
lzsa_match *pMatch = pCompressor->best_match + i;
if (pMatch->length >= MIN_MATCH_SIZE_V1) {
int nMatchOffset = pMatch->offset;

30
src/shrink_context.c
View File

@ -58,7 +58,6 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
pCompressor->intervals = NULL;
pCompressor->pos_data = NULL;
pCompressor->open_intervals = NULL;
pCompressor->match = NULL;
pCompressor->best_match = NULL;
pCompressor->arrival = NULL;
pCompressor->min_match_size = nMinMatchSize;
@ -81,22 +80,16 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
pCompressor->open_intervals = (unsigned int *)malloc((LCP_AND_TAG_MAX + 1) * sizeof(unsigned int));
if (pCompressor->open_intervals) {
pCompressor->match = (lzsa_match *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(lzsa_match));
pCompressor->arrival = (lzsa_arrival *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(lzsa_arrival));
if (pCompressor->match) {
pCompressor->arrival = (lzsa_arrival *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(lzsa_arrival));
if (pCompressor->arrival) {
pCompressor->best_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match));
if (pCompressor->arrival) {
if (pCompressor->format_version == 2) {
pCompressor->best_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match));
if (pCompressor->best_match) {
return 0;
}
}
else {
return 0;
}
if (pCompressor->best_match) {
return 0;
}
else {
return 0;
}
}
}
@ -126,11 +119,6 @@ void lzsa_compressor_destroy(lzsa_compressor *pCompressor) {
pCompressor->best_match = NULL;
}
if (pCompressor->match) {
free(pCompressor->match);
pCompressor->match = NULL;
}
if (pCompressor->open_intervals) {
free(pCompressor->open_intervals);
pCompressor->open_intervals = NULL;
@ -172,8 +160,6 @@ int lzsa_compressor_shrink_block(lzsa_compressor *pCompressor, unsigned char *pI
if (nPreviousBlockSize) {
lzsa_skip_matches(pCompressor, 0, nPreviousBlockSize);
}
if (pCompressor->format_version < 2)
lzsa_find_all_matches(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
if (pCompressor->format_version == 1) {
nCompressedSize = lzsa_optimize_and_write_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nInDataSize, pOutData, nMaxOutDataSize);

1
src/shrink_context.h
View File

@ -85,7 +85,6 @@ typedef struct _lzsa_compressor {
unsigned int *intervals;
unsigned int *pos_data;
unsigned int *open_intervals;
lzsa_match *match;
lzsa_match *best_match;
lzsa_arrival *arrival;
int min_match_size;