From 59eb8913942a38b98bcb9ae829b2d2ca334b6c54 Mon Sep 17 00:00:00 2001 From: Emmanuel Marty Date: Mon, 1 Jul 2019 09:25:19 +0200 Subject: [PATCH] Improve LZSA2 compression ratio further and allow incompressible raw blocks --- Makefile | 1 + Xcode/lzsa.xcodeproj/project.pbxproj | 6 + src/hashmap.c | 138 +++++++++ src/hashmap.h | 99 +++++++ src/lzsa.c | 4 +- src/shrink_block_v1.c | 53 +++- src/shrink_block_v2.c | 408 ++++++++++++++++++++++++--- src/shrink_context.c | 35 ++- src/shrink_context.h | 5 + src/shrink_inmem.c | 15 + src/shrink_streaming.c | 42 ++- 11 files changed, 760 insertions(+), 46 deletions(-) create mode 100644 src/hashmap.c create mode 100644 src/hashmap.h diff --git a/Makefile b/Makefile index 9e98565..cf84d1b 100755 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ OBJS += $(OBJDIR)/src/expand_context.o OBJS += $(OBJDIR)/src/expand_inmem.o OBJS += $(OBJDIR)/src/expand_streaming.o OBJS += $(OBJDIR)/src/frame.o +OBJS += $(OBJDIR)/src/hashmap.o OBJS += $(OBJDIR)/src/matchfinder.o OBJS += $(OBJDIR)/src/shrink_block_v1.o OBJS += $(OBJDIR)/src/shrink_block_v2.o diff --git a/Xcode/lzsa.xcodeproj/project.pbxproj b/Xcode/lzsa.xcodeproj/project.pbxproj index 608a7d6..e5ec4a9 100644 --- a/Xcode/lzsa.xcodeproj/project.pbxproj +++ b/Xcode/lzsa.xcodeproj/project.pbxproj @@ -26,6 +26,7 @@ 0CADC64722AAD8EB003E9821 /* expand_context.c in Sources */ = {isa = PBXBuildFile; fileRef = 0CADC62F22AAD8EB003E9821 /* expand_context.c */; }; 0CADC64822AAD8EB003E9821 /* shrink_block_v2.c in Sources */ = {isa = PBXBuildFile; fileRef = 0CADC63022AAD8EB003E9821 /* shrink_block_v2.c */; }; 0CADC64A22AB8DAD003E9821 /* divsufsort_utils.c in Sources */ = {isa = PBXBuildFile; fileRef = 0CADC64922AB8DAD003E9821 /* divsufsort_utils.c */; }; + 0CADC69622C8A420003E9821 /* hashmap.c in Sources */ = {isa = PBXBuildFile; fileRef = 0CADC69522C8A41F003E9821 /* hashmap.c */; }; /* End PBXBuildFile section */ /* Begin PBXCopyFilesBuildPhase section */ @@ -80,6 +81,8 @@ 0CADC63022AAD8EB003E9821 /* shrink_block_v2.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = shrink_block_v2.c; path = ../../src/shrink_block_v2.c; sourceTree = ""; }; 0CADC64922AB8DAD003E9821 /* divsufsort_utils.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = divsufsort_utils.c; sourceTree = ""; }; 0CADC64B22AB8DC3003E9821 /* divsufsort_config.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = divsufsort_config.h; sourceTree = ""; }; + 0CADC69422C8A41F003E9821 /* hashmap.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = hashmap.h; path = ../../src/hashmap.h; sourceTree = ""; }; + 0CADC69522C8A41F003E9821 /* hashmap.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = hashmap.c; path = ../../src/hashmap.c; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -127,6 +130,8 @@ 0CADC62422AAD8EB003E9821 /* format.h */, 0CADC5F322AAD8EB003E9821 /* frame.c */, 0CADC62C22AAD8EB003E9821 /* frame.h */, + 0CADC69522C8A41F003E9821 /* hashmap.c */, + 0CADC69422C8A41F003E9821 /* hashmap.h */, 0CADC5F222AAD8EB003E9821 /* lib.h */, 0CADC5FC22AAD8EB003E9821 /* libdivsufsort */, 0CADC62222AAD8EB003E9821 /* lzsa.c */, @@ -235,6 +240,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 0CADC69622C8A420003E9821 /* hashmap.c in Sources */, 0CADC64822AAD8EB003E9821 /* shrink_block_v2.c in Sources */, 0CADC63D22AAD8EB003E9821 /* sssort.c in Sources */, 0CADC64322AAD8EB003E9821 /* expand_block_v2.c in Sources */, diff --git a/src/hashmap.c b/src/hashmap.c new file mode 100644 index 0000000..71bf01b --- /dev/null +++ b/src/hashmap.c @@ -0,0 +1,138 @@ +/* + * hashmap.c - integer hashmap implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "hashmap.h" + +/** + * Generate key hash by mixing + * + * @param key key to get hash for + * + * @return hash + */ +static unsigned int lzsa_hashmap_get_hash(unsigned long long key) { + key = (~key) + (key << 21); + key = key ^ (key >> 24); + key = (key + (key << 3)) + (key << 8); + key = key ^ (key >> 14); + key = (key + (key << 2)) + (key << 4); + key = key ^ (key >> 28); + key = key + (key << 31); + return key & (LZSA_HASH_NBUCKETS - 1); +} + +/** + * Initialize hashmap + * + * @param pHashMap hashmap + */ +void lzsa_hashmap_init(lzsa_hashmap_t *pHashMap) { + pHashMap->pBuffer = NULL; + memset(pHashMap->pBucket, 0, sizeof(lzsa_hashvalue_t *) * LZSA_HASH_NBUCKETS); +} + +/** + * Set value for key + * + * @param pHashMap hashmap + * @param key key to set value for + * @param value new value + */ +void lzsa_hashmap_insert(lzsa_hashmap_t *pHashMap, unsigned long long key, unsigned int value) { + unsigned int hash = lzsa_hashmap_get_hash(key); + lzsa_hashvalue_t **pBucket = &pHashMap->pBucket[hash]; + while (*pBucket) { + if ((*pBucket)->key == key) { + (*pBucket)->value = value; + return; + } + + pBucket = &((*pBucket)->pNext); + } + + if (!pHashMap->pBuffer || pHashMap->pBuffer->nFreeEntryIdx >= 255) { + lzsa_hashbuffer_t *pNewBuffer = (lzsa_hashbuffer_t *)malloc(sizeof(lzsa_hashbuffer_t)); + if (!pNewBuffer) return; + + pNewBuffer->pNext = pHashMap->pBuffer; + pNewBuffer->nFreeEntryIdx = 0; + pHashMap->pBuffer = pNewBuffer; + } + + *pBucket = &pHashMap->pBuffer->value[pHashMap->pBuffer->nFreeEntryIdx++]; + (*pBucket)->pNext = NULL; + (*pBucket)->key = key; + (*pBucket)->value = value; +} + +/** + * Get value for key + * + * @param pHashMap hashmap + * @param key key to get value for + * @param pValue pointer to where to store value if found + * + * @return 0 if found, nonzero if not found + */ +int lzsa_hashmap_find(lzsa_hashmap_t *pHashMap, unsigned long long key, unsigned int *pValue) { + unsigned int hash = lzsa_hashmap_get_hash(key); + lzsa_hashvalue_t **pBucket = &pHashMap->pBucket[hash]; + while (*pBucket) { + if ((*pBucket)->key == key) { + *pValue = (*pBucket)->value; + return 0; + } + + pBucket = &((*pBucket)->pNext); + } + + return -1; +} + +/** + * Clear hashmap + * + * @param pHashMap hashmap + */ +void lzsa_hashmap_clear(lzsa_hashmap_t *pHashMap) { + while (pHashMap->pBuffer) { + lzsa_hashbuffer_t *pCurBuffer = pHashMap->pBuffer; + pHashMap->pBuffer = pCurBuffer->pNext; + free(pCurBuffer); + pCurBuffer = NULL; + } + + memset(pHashMap->pBucket, 0, sizeof(lzsa_hashvalue_t *) * LZSA_HASH_NBUCKETS); +} + diff --git a/src/hashmap.h b/src/hashmap.h new file mode 100644 index 0000000..e16cb91 --- /dev/null +++ b/src/hashmap.h @@ -0,0 +1,99 @@ +/* + * hashmap.h - integer hashmap definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _HASHMAP_H +#define _HASHMAP_H + +#include + +/** Number of hashmap buckets */ +#define LZSA_HASH_NBUCKETS 256 + +/* Forward definitions */ +typedef struct _lzsa_hashvalue_t lzsa_hashvalue_t; +typedef struct _lzsa_hashbuffer_t lzsa_hashbuffer_t; + +/** One hashmap bucket entry */ +typedef struct _lzsa_hashvalue_t { + lzsa_hashvalue_t *pNext; + unsigned long long key; + unsigned int value; +} lzsa_hashvalue_t; + +/** One buffer storing hashmap bucket entries */ +typedef struct _lzsa_hashbuffer_t { + lzsa_hashbuffer_t *pNext; + int nFreeEntryIdx; + lzsa_hashvalue_t value[255]; +} lzsa_hashbuffer_t; + +/** Hashmap */ +typedef struct { + lzsa_hashbuffer_t *pBuffer; + lzsa_hashvalue_t *pBucket[LZSA_HASH_NBUCKETS]; +} lzsa_hashmap_t; + +/** + * Initialize hashmap + * + * @param pHashMap hashmap + */ +void lzsa_hashmap_init(lzsa_hashmap_t *pHashMap); + +/** + * Set value for key + * + * @param pHashMap hashmap + * @param key key to set value for + * @param value new value + */ +void lzsa_hashmap_insert(lzsa_hashmap_t *pHashMap, unsigned long long key, unsigned int value); + +/** + * Get value for key + * + * @param pHashMap hashmap + * @param key key to get value for + * @param pValue pointer to where to store value if found + * + * @return 0 if found, nonzero if not found + */ +int lzsa_hashmap_find(lzsa_hashmap_t *pHashMap, unsigned long long key, unsigned int *pValue); + +/** + * Clear hashmap + * + * @param pHashMap hashmap + */ +void lzsa_hashmap_clear(lzsa_hashmap_t *pHashMap); + +#endif /* _HASHMAP_H */ diff --git a/src/lzsa.c b/src/lzsa.c index 4ab941a..a925212 100755 --- a/src/lzsa.c +++ b/src/lzsa.c @@ -46,7 +46,7 @@ #define OPT_RAW 2 #define OPT_FAVOR_RATIO 4 -#define TOOL_VERSION "1.0.2" +#define TOOL_VERSION "1.0.3" /*---------------------------------------------------------------------------*/ @@ -127,7 +127,7 @@ static int do_compress(const char *pszInFilename, const char *pszOutFilename, co case LZSA_ERROR_MEMORY: fprintf(stderr, "out of memory\n"); break; case LZSA_ERROR_COMPRESSION: fprintf(stderr, "internal compression error\n"); break; case LZSA_ERROR_RAW_TOOLARGE: fprintf(stderr, "error: raw blocks can only be used with files <= 64 Kb\n"); break; - case LZSA_ERROR_RAW_UNCOMPRESSED: fprintf(stderr, "error: data is incompressible, raw blocks only support compressed data\n"); break; + case LZSA_ERROR_RAW_UNCOMPRESSED: fprintf(stderr, "error: incompressible data needs to be <= 64 Kb in raw blocks\n"); break; case LZSA_OK: break; default: fprintf(stderr, "unknown compression error %d\n", nStatus); break; } diff --git a/src/shrink_block_v1.c b/src/shrink_block_v1.c index ad90cd2..6fae63d 100644 --- a/src/shrink_block_v1.c +++ b/src/shrink_block_v1.c @@ -433,6 +433,50 @@ static int lzsa_write_block_v1(lzsa_compressor *pCompressor, const unsigned char return nOutOffset; } +/** + * Emit raw block of uncompressible data + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +static int lzsa_write_raw_uncompressed_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) { + int nNumLiterals = nEndOffset - nStartOffset; + int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V1) ? LITERALS_RUN_LEN_V1 : nNumLiterals; + int nOutOffset = 0; + + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3) + 4; + if ((nOutOffset + (nCommandSize >> 3)) > nMaxOutDataSize) + return -1; + + pCompressor->num_commands = 0; + pOutData[nOutOffset++] = (nTokenLiteralsLen << 4) | 0x0f; + + nOutOffset = lzsa_write_literals_varlen_v1(pOutData, nOutOffset, nNumLiterals); + + if (nNumLiterals != 0) { + memcpy(pOutData + nOutOffset, pInWindow + nStartOffset, nNumLiterals); + nOutOffset += nNumLiterals; + nNumLiterals = 0; + } + + pCompressor->num_commands++; + + /* Emit EOD marker for raw block */ + + pOutData[nOutOffset++] = 0; + pOutData[nOutOffset++] = 238; + pOutData[nOutOffset++] = 0; + pOutData[nOutOffset++] = 0; + + return nOutOffset; +} + /** * Select the most optimal matches, reduce the token count if possible, and then emit a block of compressed LZSA1 data * @@ -446,6 +490,8 @@ static int lzsa_write_block_v1(lzsa_compressor *pCompressor, const unsigned char * @return size of compressed data in output buffer, or -1 if the data is uncompressible */ int lzsa_optimize_and_write_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) { + int nResult; + lzsa_optimize_matches_v1(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); int nDidReduce; @@ -455,5 +501,10 @@ int lzsa_optimize_and_write_block_v1(lzsa_compressor *pCompressor, const unsigne nPasses++; } while (nDidReduce && nPasses < 20); - return lzsa_write_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); + nResult = lzsa_write_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); + if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + nResult = lzsa_write_raw_uncompressed_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); + } + + return nResult; } diff --git a/src/shrink_block_v2.c b/src/shrink_block_v2.c index f398c91..ef71155 100644 --- a/src/shrink_block_v2.c +++ b/src/shrink_block_v2.c @@ -35,6 +35,9 @@ #include "lib.h" #include "shrink_block_v2.h" #include "format.h" +#include "hashmap.h" + +#define HASH_KEY(__nRepMatchOffset,__nNumLiterals,__i) (((unsigned long long)(__nRepMatchOffset)) | (((unsigned long long)(__i)) << 17) | (((unsigned long long)(__nNumLiterals)) << 34)) /** * Write 4-bit nibble to output (compressed) buffer @@ -222,7 +225,7 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt if (pCompressor->best_match[i + 1].length >= MIN_MATCH_SIZE_V2) nLiteralsCost += MODESWITCH_PENALTY; - lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT); + const lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT); int *pSlotCost = pCompressor->slot_cost + (i << MATCHES_PER_OFFSET_SHIFT); int m; @@ -309,8 +312,8 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt if (nMatchOffsetSize && nCurIndex < nEndOffset && pCompressor->best_match[nCurIndex].length >= MIN_MATCH_SIZE_V2 && !repmatch_opt[nCurIndex].expected_repmatch) { int r; - for (r = 0; r < NMATCHES_PER_OFFSET && pCompressor->match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].length >= MIN_MATCH_SIZE_V2; r++) { - if (pCompressor->match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].offset == pMatch[m].offset) { + for (r = 0; r < NMATCHES_PER_OFFSET && pCompressor->selected_match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].length >= MIN_MATCH_SIZE_V2; r++) { + if (pCompressor->selected_match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].offset == pMatch[m].offset) { int nAltCost = nCurCost - nMatchOffsetSize + pCompressor->slot_cost[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r] - cost[nCurIndex]; if (nAltCost <= nCurCost) { @@ -356,8 +359,8 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt if (nMatchOffsetSize && nCurIndex < nEndOffset && pCompressor->best_match[nCurIndex].length >= MIN_MATCH_SIZE_V2 && !repmatch_opt[nCurIndex].expected_repmatch) { int r; - for (r = 0; r < NMATCHES_PER_OFFSET && pCompressor->match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].length >= MIN_MATCH_SIZE_V2; r++) { - if (pCompressor->match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].offset == pMatch[m].offset) { + for (r = 0; r < NMATCHES_PER_OFFSET && pCompressor->selected_match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].length >= MIN_MATCH_SIZE_V2; r++) { + if (pCompressor->selected_match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].offset == pMatch[m].offset) { int nAltCost = nCurCost - nMatchOffsetSize + pCompressor->slot_cost[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r] - cost[nCurIndex]; if (nAltCost <= nCurCost) { @@ -381,9 +384,9 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt } } - pSlotCost[m] = nBestCost; - pMatch[m].length = nBestMatchLen; - pMatch[m].offset = nBestMatchOffset; /* not necessary */ + pSlotCost[m] = nBestCost; + pCompressor->selected_match[(i << MATCHES_PER_OFFSET_SHIFT) + m].length = nBestMatchLen; + pCompressor->selected_match[(i << MATCHES_PER_OFFSET_SHIFT) + m].offset = nBestMatchOffset; if (m == 0 || (nBestMatchLen && cost[i] >= nBestCost)) { cost[i] = nBestCost; @@ -400,6 +403,7 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt } for (; m < NMATCHES_PER_OFFSET; m++) { pSlotCost[m] = 0; + pCompressor->selected_match[(i << MATCHES_PER_OFFSET_SHIFT) + m] = pMatch[m]; } if (pCompressor->best_match[i].length >= MIN_MATCH_SIZE_V2) @@ -412,7 +416,7 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt for (i = nStartOffset; i < nEndOffset; ) { if (pCompressor->best_match[i].length >= MIN_MATCH_SIZE_V2) { if (nIncomingOffset >= 0 && repmatch_opt[i].incoming_offset == nIncomingOffset && repmatch_opt[i].best_slot_for_incoming >= 0) { - lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT) + repmatch_opt[i].best_slot_for_incoming; + lzsa_match *pMatch = pCompressor->selected_match + (i << MATCHES_PER_OFFSET_SHIFT) + repmatch_opt[i].best_slot_for_incoming; int *pSlotCost = pCompressor->slot_cost + (i << MATCHES_PER_OFFSET_SHIFT) + repmatch_opt[i].best_slot_for_incoming; pCompressor->best_match[i].length = pMatch->length; @@ -446,7 +450,7 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt * * @return non-zero if the number of tokens was reduced, 0 if it wasn't */ -static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) { +static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) { int i; int nNumLiterals = 0; int nDidReduce = 0; @@ -455,7 +459,7 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in lzsa_repmatch_opt *repmatch_opt = pCompressor->repmatch_opt; for (i = nStartOffset; i < nEndOffset; ) { - lzsa_match *pMatch = pCompressor->best_match + i; + lzsa_match *pMatch = pBestMatch + i; if (pMatch->length >= MIN_MATCH_SIZE_V2) { int nMatchLen = pMatch->length; @@ -468,10 +472,10 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in int nRepMatchSize = (nRepMatchOffset <= 32) ? 4 : ((nRepMatchOffset <= 512) ? 8 : ((nRepMatchOffset <= (8192 + 512)) ? 12 : 16)) /* match offset */; int nUndoRepMatchCost = (nPreviousMatchOffset < 0 || !repmatch_opt[nPreviousMatchOffset].expected_repmatch) ? 0 : nRepMatchSize; - if (pCompressor->best_match[i + nMatchLen].length >= MIN_MATCH_SIZE_V2) { + if (pBestMatch[i + nMatchLen].length >= MIN_MATCH_SIZE_V2) { int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + lzsa_get_match_varlen_size_v2(nEncodedMatchLen) - nUndoRepMatchCost; - if (pCompressor->best_match[i + nMatchLen].offset != nMatchOffset) { + if (pBestMatch[i + nMatchLen].offset != nMatchOffset) { nCommandSize += (nMatchOffset <= 32) ? 4 : ((nMatchOffset <= 512) ? 8 : ((nMatchOffset <= (8192 + 512)) ? 12 : 16)) /* match offset */; } @@ -484,7 +488,7 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in } else { if (nMatchOffset != nRepMatchOffset && - pCompressor->best_match[i + nMatchLen].offset == nRepMatchOffset) { + pBestMatch[i + nMatchLen].offset == nRepMatchOffset) { if (nCommandSize > ((nMatchLen << 3) + lzsa_get_literals_varlen_size_v2(nNumLiterals + nMatchLen) - nRepMatchSize)) { /* Same case, replacing this command by literals alone isn't enough on its own to have savings, however this match command is inbetween two matches with @@ -503,10 +507,10 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in do { nCurIndex++; nNextNumLiterals++; - } while (nCurIndex < nEndOffset && pCompressor->best_match[nCurIndex].length < MIN_MATCH_SIZE_V2); + } while (nCurIndex < nEndOffset && pBestMatch[nCurIndex].length < MIN_MATCH_SIZE_V2); - if (nCurIndex >= nEndOffset || pCompressor->best_match[nCurIndex].length < MIN_MATCH_SIZE_V2 || - pCompressor->best_match[nCurIndex].offset != nMatchOffset) { + if (nCurIndex >= nEndOffset || pBestMatch[nCurIndex].length < MIN_MATCH_SIZE_V2 || + pBestMatch[nCurIndex].offset != nMatchOffset) { nCommandSize += (nMatchOffset <= 32) ? 4 : ((nMatchOffset <= 512) ? 8 : ((nMatchOffset <= (8192 + 512)) ? 12 : 16)) /* match offset */; } @@ -516,9 +520,9 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in nReduce = 1; } else { - if (nCurIndex < nEndOffset && pCompressor->best_match[nCurIndex].length >= MIN_MATCH_SIZE_V2 && - pCompressor->best_match[nCurIndex].offset != nMatchOffset && - pCompressor->best_match[nCurIndex].offset == nRepMatchOffset) { + if (nCurIndex < nEndOffset && pBestMatch[nCurIndex].length >= MIN_MATCH_SIZE_V2 && + pBestMatch[nCurIndex].offset != nMatchOffset && + pBestMatch[nCurIndex].offset == nRepMatchOffset) { if (nCommandSize > ((nMatchLen << 3) + lzsa_get_literals_varlen_size_v2(nNumLiterals + nNextNumLiterals + nMatchLen) - lzsa_get_literals_varlen_size_v2(nNextNumLiterals) - nRepMatchSize)) { /* Same case, but now replacing this command allows to use a rep-match and get savings, so do it */ nReduce = 1; @@ -532,7 +536,7 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in int j; for (j = 0; j < nMatchLen; j++) { - pCompressor->best_match[i + j].length = 0; + pBestMatch[i + j].length = 0; } nNumLiterals += nMatchLen; i += nMatchLen; @@ -549,13 +553,13 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in nRepMatchOffset = pMatch->offset; if ((i + nMatchLen) < nEndOffset && nMatchLen >= LCP_MAX && - pMatch->offset && pMatch->offset <= 32 && pCompressor->best_match[i + nMatchLen].offset == pMatch->offset && (nMatchLen % pMatch->offset) == 0 && - (nMatchLen + pCompressor->best_match[i + nMatchLen].length) <= MAX_VARLEN) { + pMatch->offset && pMatch->offset <= 32 && pBestMatch[i + nMatchLen].offset == pMatch->offset && (nMatchLen % pMatch->offset) == 0 && + (nMatchLen + pBestMatch[i + nMatchLen].length) <= MAX_VARLEN) { /* Join */ - pMatch->length += pCompressor->best_match[i + nMatchLen].length; - pCompressor->best_match[i + nMatchLen].offset = 0; - pCompressor->best_match[i + nMatchLen].length = -1; + pMatch->length += pBestMatch[i + nMatchLen].length; + pBestMatch[i + nMatchLen].offset = 0; + pBestMatch[i + nMatchLen].length = -1; continue; } @@ -574,10 +578,252 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in return nDidReduce; } +/** + * Get cost of the best encoding choice at a given offset, going forward + * + * @param pCompressor compression context + * @param i offset in input window + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes) + * @param nNumLiterals current pending number of literals to be encoded with the next token + * @param nRepMatchOffset current rep-match offset + * @param nDepth current recursion depth + * @param pBestMatchLen pointer to returned best match length for the position (0 for no match) + * @param pBestMatchOffset pointer to returned best match offset for the position (if there is a match) + * + * @return cost of best encoding choice for offset + */ +static int lzsa_get_forward_cost_v2(lzsa_compressor *pCompressor, const int i, const int nEndOffset, const int nNumLiterals, const int nRepMatchOffset, int nDepth, int *pBestMatchLen, int *pBestMatchOffset) { + if (i >= nEndOffset) + return 0; + + int *cost = (int*)pCompressor->pos_data; /* Reuse */ + if (nDepth >= pCompressor->max_forward_depth) + return cost[i]; + + if (nDepth >= 1) { + unsigned int nValue = 0; + + if (!lzsa_hashmap_find(&pCompressor->cost_map, HASH_KEY(nRepMatchOffset, nNumLiterals, i), &nValue)) + return nValue; + } + + int nMinMatchSize = pCompressor->min_match_size; + int m; + const lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT); + + int nBestCost, nBestMatchLen, nBestMatchOffset, nTmpMatchLen, nTmpMatchOffset; + int nLiteralsCost; + + nBestCost = 8 + lzsa_get_forward_cost_v2(pCompressor, i + 1, nEndOffset, nNumLiterals + 1, nRepMatchOffset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset); + nBestMatchLen = 0; + nBestMatchOffset = 0; + + nLiteralsCost = lzsa_get_literals_varlen_size_v2(nNumLiterals); + + for (m = 0; m < NMATCHES_PER_OFFSET && pMatch[m].length >= nMinMatchSize; m++) { + if (pMatch[m].length > 30) { + int nCurCost; + int nMatchLen = pMatch[m].length; + + if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS)) + nMatchLen = nEndOffset - LAST_LITERALS - i; + + int nMatchOffsetSize; + if (nRepMatchOffset == pMatch[m].offset) + nMatchOffsetSize = 0; + else { + nMatchOffsetSize = (pMatch[m].offset <= 32) ? 4 : ((pMatch[m].offset <= 512) ? 8 : ((pMatch[m].offset <= (8192 + 512)) ? 12 : 16)); + } + + nCurCost = 8 + nLiteralsCost + nMatchOffsetSize + lzsa_get_match_varlen_size_v2(nMatchLen - MIN_MATCH_SIZE_V2); + nCurCost += lzsa_get_forward_cost_v2(pCompressor, i + nMatchLen, nEndOffset, 0, pMatch[m].offset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset); + + if (nBestCost >= nCurCost) { + nBestCost = nCurCost; + nBestMatchLen = nMatchLen; + nBestMatchOffset = pMatch[m].offset; + } + } + else { + int nMatchLen = pMatch[m].length; + int k, nMatchRunLen; + + if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS)) + nMatchLen = nEndOffset - LAST_LITERALS - i; + + nMatchRunLen = nMatchLen; + if (nMatchRunLen > MATCH_RUN_LEN_V2) + nMatchRunLen = MATCH_RUN_LEN_V2; + + for (k = nMinMatchSize; k < nMatchRunLen; k++) { + int nCurCost; + + int nMatchOffsetSize; + if (nRepMatchOffset == pMatch[m].offset) + nMatchOffsetSize = 0; + else { + nMatchOffsetSize = (pMatch[m].offset <= 32) ? 4 : ((pMatch[m].offset <= 512) ? 8 : ((pMatch[m].offset <= (8192 + 512)) ? 12 : 16)); + } + + nCurCost = 8 + nLiteralsCost + nMatchOffsetSize /* no extra match len bytes */; + nCurCost += lzsa_get_forward_cost_v2(pCompressor, i + k, nEndOffset, 0, pMatch[m].offset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset); + + if (nBestCost >= nCurCost) { + nBestCost = nCurCost; + nBestMatchLen = k; + nBestMatchOffset = pMatch[m].offset; + } + } + + for (; k <= nMatchLen; k++) { + int nCurCost; + + int nMatchOffsetSize; + if (nRepMatchOffset == pMatch[m].offset) + nMatchOffsetSize = 0; + else { + nMatchOffsetSize = (pMatch[m].offset <= 32) ? 4 : ((pMatch[m].offset <= 512) ? 8 : ((pMatch[m].offset <= (8192 + 512)) ? 12 : 16)); + } + + nCurCost = 8 + nLiteralsCost + nMatchOffsetSize + lzsa_get_match_varlen_size_v2(k - MIN_MATCH_SIZE_V2); + nCurCost += lzsa_get_forward_cost_v2(pCompressor, i + k, nEndOffset, 0, pMatch[m].offset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset); + + if (nBestCost >= nCurCost) { + nBestCost = nCurCost; + nBestMatchLen = k; + nBestMatchOffset = pMatch[m].offset; + } + } + } + } + + *pBestMatchLen = nBestMatchLen; + *pBestMatchOffset = nBestMatchOffset; + + lzsa_hashmap_insert(&pCompressor->cost_map, HASH_KEY(nRepMatchOffset, nNumLiterals, i), nBestCost); + return nBestCost; +} + +/** + * Attempt to further improve the selected optimal matches with a chain-N forward parser pass + * + * @param pCompressor compression context + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + */ +static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) { + int i; + int nNumLiterals = 0; + int nRepMatchOffset = 0; + int *cost = (int*)pCompressor->pos_data; /* Reuse */ + + lzsa_hashmap_init(&pCompressor->cost_map); + + for (i = nStartOffset; i < nEndOffset; ) { + int nBestMatchLen = 0, nBestMatchOffset = 0; + int nBestCost = lzsa_get_forward_cost_v2(pCompressor, i, nEndOffset, nNumLiterals, nRepMatchOffset, 0, &nBestMatchLen, &nBestMatchOffset); + + lzsa_hashmap_clear(&pCompressor->cost_map); + + lzsa_match *pMatch = pCompressor->improved_match + i; + if (nBestCost < cost[i]) { + pMatch->length = nBestMatchLen; + pMatch->offset = nBestMatchOffset; + } + + if (pMatch->length >= MIN_MATCH_SIZE_V2) { + nNumLiterals = 0; + nRepMatchOffset = pMatch->offset; + + i += pMatch->length; + } + else { + nNumLiterals++; + i++; + } + } + + lzsa_hashmap_clear(&pCompressor->cost_map); +} + +/** + * Calculate compressed size + * + * @param pCompressor compression context + * @param pBestMatch optimal matches to evaluate + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * + * @return compressed size + */ +static int lzsa_get_compressed_size_v2(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) { + int i; + int nNumLiterals = 0; + int nCompressedSize = 0; + int nCurNibbleOffset = -1, nCurFreeNibbles = 0; + int nRepMatchOffset = 0; + + for (i = nStartOffset; i < nEndOffset; ) { + const lzsa_match *pMatch = pBestMatch + i; + + if (pMatch->length >= MIN_MATCH_SIZE_V2) { + int nMatchOffset = pMatch->offset; + int nMatchLen = pMatch->length; + int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE_V2; + int nOffsetSize; + + if (nMatchOffset == nRepMatchOffset) { + nOffsetSize = 0; + } + else { + if (nMatchOffset <= 32) { + nOffsetSize = 4; + } + else if (nMatchOffset <= 512) { + nOffsetSize = 8; + } + else if (nMatchOffset <= (8192 + 512)) { + nOffsetSize = 12; + } + else { + nOffsetSize = 16; + } + } + + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3) + nOffsetSize /* match offset */ + lzsa_get_match_varlen_size_v2(nEncodedMatchLen); + nCompressedSize += nCommandSize; + + nNumLiterals = 0; + nRepMatchOffset = nMatchOffset; + + i += nMatchLen; + } + else { + nNumLiterals++; + i++; + } + } + + { + int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V2) ? LITERALS_RUN_LEN_V2 : nNumLiterals; + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3); + + nCompressedSize += nCommandSize; + nNumLiterals = 0; + } + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + nCompressedSize += 8 + 4 + 8; + } + + return nCompressedSize; +} + /** * Emit block of compressed data * * @param pCompressor compression context + * @param pBestMatch optimal matches to emit * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes @@ -586,7 +832,7 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in * * @return size of compressed data in output buffer, or -1 if the data is uncompressible */ -static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) { +static int lzsa_write_block_v2(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) { int i; int nNumLiterals = 0; int nInFirstLiteralOffset = 0; @@ -595,7 +841,7 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const unsigned char int nRepMatchOffset = 0; for (i = nStartOffset; i < nEndOffset; ) { - lzsa_match *pMatch = pCompressor->best_match + i; + const lzsa_match *pMatch = pBestMatch + i; if (pMatch->length >= MIN_MATCH_SIZE_V2) { int nMatchOffset = pMatch->offset; @@ -727,6 +973,63 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const unsigned char return nOutOffset; } +/** + * Emit raw block of uncompressible data + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +static int lzsa_write_raw_uncompressed_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) { + int nCurNibbleOffset = -1, nCurFreeNibbles = 0; + int nNumLiterals = nEndOffset - nStartOffset; + int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V2) ? LITERALS_RUN_LEN_V2 : nNumLiterals; + int nOutOffset = 0; + + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3) + 8 + 4 + 8; + if ((nOutOffset + ((nCommandSize + 7) >> 3)) > nMaxOutDataSize) + return -1; + + pCompressor->num_commands = 0; + pOutData[nOutOffset++] = (nTokenLiteralsLen << 3) | 0x47; + + nOutOffset = lzsa_write_literals_varlen_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, &nCurFreeNibbles, nNumLiterals); + if (nOutOffset < 0) return -1; + + if (nNumLiterals != 0) { + memcpy(pOutData + nOutOffset, pInWindow + nStartOffset, nNumLiterals); + nOutOffset += nNumLiterals; + nNumLiterals = 0; + } + + /* Emit EOD marker for raw block */ + + pOutData[nOutOffset++] = 0; /* Match offset */ + + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, &nCurFreeNibbles, 15); /* Extended match length nibble */ + if (nOutOffset < 0) return -1; + + if ((nOutOffset + 1) > nMaxOutDataSize) + return -1; + + pOutData[nOutOffset++] = 232; /* EOD match length byte */ + + pCompressor->num_commands++; + + if (nCurNibbleOffset != -1) { + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, &nCurFreeNibbles, 0); + if (nOutOffset < 0 || nCurNibbleOffset != -1) + return -1; + } + + return nOutOffset; +} + /** * Select the most optimal matches, reduce the token count if possible, and then emit a block of compressed LZSA2 data * @@ -740,14 +1043,49 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const unsigned char * @return size of compressed data in output buffer, or -1 if the data is uncompressible */ int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) { + int nResult; + lzsa_optimize_matches_v2(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + lzsa_match *pBestMatch; + if (pCompressor->max_forward_depth > 0) { + memcpy(pCompressor->improved_match, pCompressor->best_match, nInDataSize * sizeof(lzsa_match)); + lzsa_optimize_forward_v2(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); - int nDidReduce; - int nPasses = 0; - do { - nDidReduce = lzsa_optimize_command_count_v2(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); - nPasses++; - } while (nDidReduce && nPasses < 20); + int nDidReduce; + int nPasses = 0; + do { + nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + nPasses++; + } while (nDidReduce && nPasses < 20); - return lzsa_write_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); + nPasses = 0; + do { + nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pCompressor->improved_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + nPasses++; + } while (nDidReduce && nPasses < 20); + + int nBestCost = lzsa_get_compressed_size_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + int nImprovedCost = lzsa_get_compressed_size_v2(pCompressor, pCompressor->improved_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + if (nBestCost > nImprovedCost) + pBestMatch = pCompressor->improved_match; + else + pBestMatch = pCompressor->best_match; + } + else { + int nDidReduce; + int nPasses = 0; + do { + nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + nPasses++; + } while (nDidReduce && nPasses < 20); + + pBestMatch = pCompressor->best_match; + } + + nResult = lzsa_write_block_v2(pCompressor, pBestMatch, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); + if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + nResult = lzsa_write_raw_uncompressed_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); + } + + return nResult; } diff --git a/src/shrink_context.c b/src/shrink_context.c index 1f5e8cc..2ccce8e 100644 --- a/src/shrink_context.c +++ b/src/shrink_context.c @@ -58,7 +58,9 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize, pCompressor->pos_data = NULL; pCompressor->open_intervals = NULL; pCompressor->match = NULL; + pCompressor->selected_match = NULL; pCompressor->best_match = NULL; + pCompressor->improved_match = NULL; pCompressor->slot_cost = NULL; pCompressor->repmatch_opt = NULL; pCompressor->min_match_size = nMinMatchSize; @@ -66,6 +68,7 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize, pCompressor->min_match_size = nMinMatchSizeForFormat; else if (pCompressor->min_match_size > nMaxMinMatchForFormat) pCompressor->min_match_size = nMaxMinMatchForFormat; + pCompressor->max_forward_depth = 0; pCompressor->format_version = nFormatVersion; pCompressor->flags = nFlags; pCompressor->num_commands = 0; @@ -84,16 +87,24 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize, if (pCompressor->match) { if (pCompressor->format_version == 2) { - pCompressor->best_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match)); + pCompressor->selected_match = (lzsa_match *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(lzsa_match)); - if (pCompressor->best_match) { - pCompressor->slot_cost = (int *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(int)); + if (pCompressor->selected_match) { + pCompressor->best_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match)); - if (pCompressor->slot_cost) { - pCompressor->repmatch_opt = (lzsa_repmatch_opt *)malloc(nMaxWindowSize * sizeof(lzsa_repmatch_opt)); + if (pCompressor->best_match) { + pCompressor->improved_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match)); - if (pCompressor->repmatch_opt) - return 0; + if (pCompressor->improved_match) { + pCompressor->slot_cost = (int *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(int)); + + if (pCompressor->slot_cost) { + pCompressor->repmatch_opt = (lzsa_repmatch_opt *)malloc(nMaxWindowSize * sizeof(lzsa_repmatch_opt)); + + if (pCompressor->repmatch_opt) + return 0; + } + } } } } @@ -128,11 +139,21 @@ void lzsa_compressor_destroy(lzsa_compressor *pCompressor) { pCompressor->slot_cost = NULL; } + if (pCompressor->improved_match) { + free(pCompressor->improved_match); + pCompressor->improved_match = NULL; + } + if (pCompressor->best_match) { free(pCompressor->best_match); pCompressor->best_match = NULL; } + if (pCompressor->selected_match) { + free(pCompressor->selected_match); + pCompressor->selected_match = NULL; + } + if (pCompressor->match) { free(pCompressor->match); pCompressor->match = NULL; diff --git a/src/shrink_context.h b/src/shrink_context.h index 7360044..46c695c 100644 --- a/src/shrink_context.h +++ b/src/shrink_context.h @@ -34,6 +34,7 @@ #define _SHRINK_CONTEXT_H #include "divsufsort.h" +#include "hashmap.h" #ifdef __cplusplus extern "C" { @@ -77,13 +78,17 @@ typedef struct _lzsa_compressor { unsigned int *pos_data; unsigned int *open_intervals; lzsa_match *match; + lzsa_match *selected_match; lzsa_match *best_match; + lzsa_match *improved_match; int *slot_cost; lzsa_repmatch_opt *repmatch_opt; int min_match_size; + int max_forward_depth; int format_version; int flags; int num_commands; + lzsa_hashmap_t cost_map; } lzsa_compressor; /** diff --git a/src/shrink_inmem.c b/src/shrink_inmem.c index aecc2ed..32a727b 100644 --- a/src/shrink_inmem.c +++ b/src/shrink_inmem.c @@ -84,6 +84,21 @@ size_t lzsa_compress_inmem(const unsigned char *pInputData, unsigned char *pOutB } } + if ((compressor.flags & LZSA_FLAG_FAVOR_RATIO)) { + if (nInputSize < 16384) + compressor.max_forward_depth = 25; + else { + if (nInputSize < 32768) + compressor.max_forward_depth = 15; + else { + if (nInputSize < BLOCK_SIZE) + compressor.max_forward_depth = 10; + else + compressor.max_forward_depth = 0; + } + } + } + int nPreviousBlockSize = 0; int nNumBlocks = 0; diff --git a/src/shrink_streaming.c b/src/shrink_streaming.c index b09ab13..37ba0f3 100644 --- a/src/shrink_streaming.c +++ b/src/shrink_streaming.c @@ -36,6 +36,24 @@ #include "format.h" #include "frame.h" #include "lib.h" +#ifdef _WIN32 +#include +#else +#include +#endif + +/** + * Delete file + * + * @param pszInFilename name of file to delete + */ +static void lzsa_delete_file(const char *pszInFilename) { +#ifdef _WIN32 + DeleteFileA(pszInFilename); +#else + remove(pszInFilename); +#endif +} /*-------------- File API -------------- */ @@ -76,6 +94,7 @@ lzsa_status_t lzsa_compress_file(const char *pszInFilename, const char *pszOutFi if (nStatus) { outStream.close(&outStream); inStream.close(&inStream); + lzsa_delete_file(pszOutFilename); return nStatus; } @@ -84,6 +103,11 @@ lzsa_status_t lzsa_compress_file(const char *pszInFilename, const char *pszOutFi lzsa_dictionary_free(&pDictionaryData); outStream.close(&outStream); inStream.close(&inStream); + + if (nStatus) { + lzsa_delete_file(pszOutFilename); + } + return nStatus; } @@ -115,6 +139,7 @@ lzsa_status_t lzsa_compress_stream(lzsa_stream_t *pInStream, lzsa_stream_t *pOut int nResult; unsigned char cFrameData[16]; int nError = 0; + int nRawPadding = (nFlags & LZSA_FLAG_RAW_BLOCK) ? 8 : 0; pInData = (unsigned char*)malloc(BLOCK_SIZE * 2); if (!pInData) { @@ -175,9 +200,24 @@ lzsa_status_t lzsa_compress_stream(lzsa_stream_t *pInStream, lzsa_stream_t *pOut } nDictionaryDataSize = 0; + if (nNumBlocks == 0 && (compressor.flags & LZSA_FLAG_FAVOR_RATIO)) { + if (nInDataSize < 16384) + compressor.max_forward_depth = 25; + else { + if (nInDataSize < 32768) + compressor.max_forward_depth = 15; + else { + if (nInDataSize < BLOCK_SIZE) + compressor.max_forward_depth = 10; + else + compressor.max_forward_depth = 0; + } + } + } + int nOutDataSize; - nOutDataSize = lzsa_compressor_shrink_block(&compressor, pInData + BLOCK_SIZE - nPreviousBlockSize, nPreviousBlockSize, nInDataSize, pOutData, (nInDataSize >= BLOCK_SIZE) ? BLOCK_SIZE : nInDataSize); + nOutDataSize = lzsa_compressor_shrink_block(&compressor, pInData + BLOCK_SIZE - nPreviousBlockSize, nPreviousBlockSize, nInDataSize, pOutData, ((nInDataSize + nRawPadding) >= BLOCK_SIZE) ? BLOCK_SIZE : (nInDataSize + nRawPadding)); if (nOutDataSize >= 0) { /* Write compressed block */