From 53b2013b73050eb626124e37dc75a59a142bad8c Mon Sep 17 00:00:00 2001 From: Emmanuel Marty Date: Mon, 11 Nov 2019 18:41:08 +0100 Subject: [PATCH] Small improvement to merging large matches --- src/shrink_block_v1.c | 29 +++++++++++++++++--------- src/shrink_block_v2.c | 48 +++++++++++++++++++++++++++++++++---------- src/shrink_context.h | 2 +- 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/src/shrink_block_v1.c b/src/shrink_block_v1.c index 47bf019..8ddf661 100644 --- a/src/shrink_block_v1.c +++ b/src/shrink_block_v1.c @@ -327,24 +327,33 @@ static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const un } } - if ((i + pMatch->length) < nEndOffset && pMatch->length >= LCP_MAX && - pBestMatch[i + pMatch->length].offset && + if ((i + pMatch->length) < nEndOffset && pMatch->offset > 0 && pMatch->length >= MIN_MATCH_SIZE_V1 && + pBestMatch[i + pMatch->length].offset > 0 && pBestMatch[i + pMatch->length].length >= MIN_MATCH_SIZE_V1 && - pBestMatch[i + pMatch->length].offset <= pMatch->length && + (pMatch->length + pBestMatch[i + pMatch->length].length) >= LEAVE_ALONE_MATCH_SIZE && (pMatch->length + pBestMatch[i + pMatch->length].length) <= MAX_VARLEN && (i + pMatch->length) > pMatch->offset && - (i - pMatch->offset + pMatch->length + pBestMatch[i + pMatch->length].length) < nEndOffset && + (i + pMatch->length) > pBestMatch[i + pMatch->length].offset && + (i + pMatch->length + pBestMatch[i + pMatch->length].length) < nEndOffset && !memcmp(pInWindow + i - pMatch->offset + pMatch->length, pInWindow + i + pMatch->length - pBestMatch[i + pMatch->length].offset, pBestMatch[i + pMatch->length].length)) { - int nMatchLen = pMatch->length; - /* Join */ + int nCurPartialSize = lzsa_get_match_varlen_size_v1(pMatch->length - MIN_MATCH_SIZE_V1); + nCurPartialSize += 8 /* token */ + lzsa_get_literals_varlen_size_v1(0) + ((pBestMatch[i + pMatch->length].offset <= 256) ? 8 : 16) /* match offset */ + lzsa_get_match_varlen_size_v1(pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V1); - pMatch->length += pBestMatch[i + nMatchLen].length; - pBestMatch[i + nMatchLen].offset = 0; - pBestMatch[i + nMatchLen].length = -1; - continue; + int nReducedPartialSize = lzsa_get_match_varlen_size_v1(pMatch->length + pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V1); + + if (nCurPartialSize >= nReducedPartialSize) { + int nMatchLen = pMatch->length; + + /* Join */ + + pMatch->length += pBestMatch[i + nMatchLen].length; + pBestMatch[i + nMatchLen].offset = 0; + pBestMatch[i + nMatchLen].length = -1; + continue; + } } i += pMatch->length; diff --git a/src/shrink_block_v2.c b/src/shrink_block_v2.c index b06b01f..7ad99f7 100644 --- a/src/shrink_block_v2.c +++ b/src/shrink_block_v2.c @@ -611,25 +611,51 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const un } } - if ((i + pMatch->length) < nEndOffset && pMatch->length >= LCP_MAX && - pBestMatch[i + pMatch->length].offset && + if ((i + pMatch->length) < nEndOffset && pMatch->offset > 0 && pMatch->length >= MIN_MATCH_SIZE_V2 && + pBestMatch[i + pMatch->length].offset > 0 && pBestMatch[i + pMatch->length].length >= MIN_MATCH_SIZE_V2 && - pBestMatch[i + pMatch->length].offset <= pMatch->length && + (pMatch->length + pBestMatch[i + pMatch->length].length) >= LEAVE_ALONE_MATCH_SIZE && (pMatch->length + pBestMatch[i + pMatch->length].length) <= MAX_VARLEN && (i + pMatch->length) > pMatch->offset && - (i - pMatch->offset + pMatch->length + pBestMatch[i + pMatch->length].length) < nEndOffset && + (i + pMatch->length) > pBestMatch[i + pMatch->length].offset && + (i + pMatch->length + pBestMatch[i + pMatch->length].length) < nEndOffset && !memcmp(pInWindow + i - pMatch->offset + pMatch->length, pInWindow + i + pMatch->length - pBestMatch[i + pMatch->length].offset, pBestMatch[i + pMatch->length].length)) { - int nMatchLen = pMatch->length; - /* Join */ + int nNextIndex = i + pMatch->length; + int nNextLiterals = 0; - pMatch->length += pBestMatch[i + nMatchLen].length; - pBestMatch[i + nMatchLen].offset = 0; - pBestMatch[i + nMatchLen].length = -1; - nDidReduce = 1; - continue; + while (nNextIndex < nEndOffset && pBestMatch[nNextIndex].length < MIN_MATCH_SIZE_V2) { + nNextLiterals++; + nNextIndex++; + } + + int nCurPartialSize = lzsa_get_match_varlen_size_v2(pMatch->length - MIN_MATCH_SIZE_V2); + + nCurPartialSize += 8 /* token */ + lzsa_get_literals_varlen_size_v2(0) + lzsa_get_match_varlen_size_v2(pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V2); + if (pBestMatch[i + pMatch->length].offset != pMatch->offset) + nCurPartialSize += (pBestMatch[i + pMatch->length].offset <= 32) ? 4 : ((pBestMatch[i + pMatch->length].offset <= 512) ? 8 : ((pBestMatch[i + pMatch->length].offset <= (8192 + 512)) ? 12 : 16)); + + if (pBestMatch[nNextIndex].offset != pBestMatch[i + pMatch->length].offset) + nCurPartialSize += (pBestMatch[nNextIndex].offset <= 32) ? 4 : ((pBestMatch[nNextIndex].offset <= 512) ? 8 : ((pBestMatch[nNextIndex].offset <= (8192 + 512)) ? 12 : 16)); + + int nReducedPartialSize = lzsa_get_match_varlen_size_v2(pMatch->length + pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V2); + + if (pBestMatch[nNextIndex].offset != pMatch->offset) + nReducedPartialSize += (pBestMatch[nNextIndex].offset <= 32) ? 4 : ((pBestMatch[nNextIndex].offset <= 512) ? 8 : ((pBestMatch[nNextIndex].offset <= (8192 + 512)) ? 12 : 16)); + + if (nCurPartialSize >= nReducedPartialSize) { + int nMatchLen = pMatch->length; + + /* Join */ + + pMatch->length += pBestMatch[i + nMatchLen].length; + pBestMatch[i + nMatchLen].offset = 0; + pBestMatch[i + nMatchLen].length = -1; + nDidReduce = 1; + continue; + } } nPrevRepMatchOffset = nRepMatchOffset; diff --git a/src/shrink_context.h b/src/shrink_context.h index a6a442f..ff8c06a 100644 --- a/src/shrink_context.h +++ b/src/shrink_context.h @@ -41,7 +41,7 @@ extern "C" { #define LCP_BITS 14 #define TAG_BITS 3 -#define LCP_MAX (1U<<(LCP_BITS - TAG_BITS - 1)) +#define LCP_MAX ((1U<<(LCP_BITS - TAG_BITS)) - 1) #define LCP_AND_TAG_MAX (1U<<(LCP_BITS - 1)) #define LCP_SHIFT (31-LCP_BITS) #define LCP_MASK (((1U<