Improve LZSA2 compression ratio further and allow incompressible raw blocks

Makefile

@@ -18,6 +18,7 @@ OBJS += $(OBJDIR)/src/expand_context.o
 OBJS += $(OBJDIR)/src/expand_inmem.o
 OBJS += $(OBJDIR)/src/expand_streaming.o
 OBJS += $(OBJDIR)/src/frame.o
+OBJS += $(OBJDIR)/src/hashmap.o
 OBJS += $(OBJDIR)/src/matchfinder.o
 OBJS += $(OBJDIR)/src/shrink_block_v1.o
 OBJS += $(OBJDIR)/src/shrink_block_v2.o

Xcode/lzsa.xcodeproj/project.pbxproj

@@ -26,6 +26,7 @@
 		0CADC64722AAD8EB003E9821 /* expand_context.c in Sources */ = {isa = PBXBuildFile; fileRef = 0CADC62F22AAD8EB003E9821 /* expand_context.c */; };
 		0CADC64822AAD8EB003E9821 /* shrink_block_v2.c in Sources */ = {isa = PBXBuildFile; fileRef = 0CADC63022AAD8EB003E9821 /* shrink_block_v2.c */; };
 		0CADC64A22AB8DAD003E9821 /* divsufsort_utils.c in Sources */ = {isa = PBXBuildFile; fileRef = 0CADC64922AB8DAD003E9821 /* divsufsort_utils.c */; };
+		0CADC69622C8A420003E9821 /* hashmap.c in Sources */ = {isa = PBXBuildFile; fileRef = 0CADC69522C8A41F003E9821 /* hashmap.c */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXCopyFilesBuildPhase section */
@@ -80,6 +81,8 @@
 		0CADC63022AAD8EB003E9821 /* shrink_block_v2.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = shrink_block_v2.c; path = ../../src/shrink_block_v2.c; sourceTree = "<group>"; };
 		0CADC64922AB8DAD003E9821 /* divsufsort_utils.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = divsufsort_utils.c; sourceTree = "<group>"; };
 		0CADC64B22AB8DC3003E9821 /* divsufsort_config.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = divsufsort_config.h; sourceTree = "<group>"; };
+		0CADC69422C8A41F003E9821 /* hashmap.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = hashmap.h; path = ../../src/hashmap.h; sourceTree = "<group>"; };
+		0CADC69522C8A41F003E9821 /* hashmap.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = hashmap.c; path = ../../src/hashmap.c; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -127,6 +130,8 @@
 				0CADC62422AAD8EB003E9821 /* format.h */,
 				0CADC5F322AAD8EB003E9821 /* frame.c */,
 				0CADC62C22AAD8EB003E9821 /* frame.h */,
+				0CADC69522C8A41F003E9821 /* hashmap.c */,
+				0CADC69422C8A41F003E9821 /* hashmap.h */,
 				0CADC5F222AAD8EB003E9821 /* lib.h */,
 				0CADC5FC22AAD8EB003E9821 /* libdivsufsort */,
 				0CADC62222AAD8EB003E9821 /* lzsa.c */,
@@ -235,6 +240,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				0CADC69622C8A420003E9821 /* hashmap.c in Sources */,
 				0CADC64822AAD8EB003E9821 /* shrink_block_v2.c in Sources */,
 				0CADC63D22AAD8EB003E9821 /* sssort.c in Sources */,
 				0CADC64322AAD8EB003E9821 /* expand_block_v2.c in Sources */,

src/hashmap.c

@@ -0,0 +1,138 @@
+/*
+ * hashmap.c - integer hashmap implementation
+ *
+ * Copyright (C) 2019 Emmanuel Marty
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty.  In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ *    claim that you wrote the original software. If you use this software
+ *    in a product, an acknowledgment in the product documentation would be
+ *    appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ *    misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori
+ *
+ * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4
+ * With help, ideas, optimizations and speed measurements by spke <zxintrospec@gmail.com>
+ * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard
+ * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/
+ *
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include "hashmap.h"
+
+/**
+ * Generate key hash by mixing
+ *
+ * @param key key to get hash for
+ *
+ * @return hash
+ */
+static unsigned int lzsa_hashmap_get_hash(unsigned long long key) {
+   key = (~key) + (key << 21);
+   key = key ^ (key >> 24);
+   key = (key + (key << 3)) + (key << 8);
+   key = key ^ (key >> 14);
+   key = (key + (key << 2)) + (key << 4);
+   key = key ^ (key >> 28);
+   key = key + (key << 31);
+   return key & (LZSA_HASH_NBUCKETS - 1);
+}
+
+/**
+ * Initialize hashmap
+ *
+ * @param pHashMap hashmap
+ */
+void lzsa_hashmap_init(lzsa_hashmap_t *pHashMap) {
+   pHashMap->pBuffer = NULL;
+   memset(pHashMap->pBucket, 0, sizeof(lzsa_hashvalue_t *) * LZSA_HASH_NBUCKETS);
+}
+
+/**
+ * Set value for key
+ *
+ * @param pHashMap hashmap
+ * @param key key to set value for
+ * @param value new value
+ */
+void lzsa_hashmap_insert(lzsa_hashmap_t *pHashMap, unsigned long long key, unsigned int value) {
+   unsigned int hash = lzsa_hashmap_get_hash(key);
+   lzsa_hashvalue_t **pBucket = &pHashMap->pBucket[hash];
+   while (*pBucket) {
+      if ((*pBucket)->key == key) {
+         (*pBucket)->value = value;
+         return;
+      }
+
+      pBucket = &((*pBucket)->pNext);
+   }
+
+   if (!pHashMap->pBuffer || pHashMap->pBuffer->nFreeEntryIdx >= 255) {
+      lzsa_hashbuffer_t *pNewBuffer = (lzsa_hashbuffer_t *)malloc(sizeof(lzsa_hashbuffer_t));
+      if (!pNewBuffer) return;
+
+      pNewBuffer->pNext = pHashMap->pBuffer;
+      pNewBuffer->nFreeEntryIdx = 0;
+      pHashMap->pBuffer = pNewBuffer;
+   }
+
+   *pBucket = &pHashMap->pBuffer->value[pHashMap->pBuffer->nFreeEntryIdx++];
+   (*pBucket)->pNext = NULL;
+   (*pBucket)->key = key;
+   (*pBucket)->value = value;
+}
+
+/**
+ * Get value for key
+ *
+ * @param pHashMap hashmap
+ * @param key key to get value for
+ * @param pValue pointer to where to store value if found
+ *
+ * @return 0 if found, nonzero if not found
+ */
+int lzsa_hashmap_find(lzsa_hashmap_t *pHashMap, unsigned long long key, unsigned int *pValue) {
+   unsigned int hash = lzsa_hashmap_get_hash(key);
+   lzsa_hashvalue_t **pBucket = &pHashMap->pBucket[hash];
+   while (*pBucket) {
+      if ((*pBucket)->key == key) {
+         *pValue = (*pBucket)->value;
+         return 0;
+      }
+
+      pBucket = &((*pBucket)->pNext);
+   }
+
+   return -1;
+}
+
+/**
+ * Clear hashmap
+ *
+ * @param pHashMap hashmap
+ */
+void lzsa_hashmap_clear(lzsa_hashmap_t *pHashMap) {
+   while (pHashMap->pBuffer) {
+      lzsa_hashbuffer_t *pCurBuffer = pHashMap->pBuffer;
+      pHashMap->pBuffer = pCurBuffer->pNext;
+      free(pCurBuffer);
+      pCurBuffer = NULL;
+   }
+
+   memset(pHashMap->pBucket, 0, sizeof(lzsa_hashvalue_t *) * LZSA_HASH_NBUCKETS);
+}
+

src/hashmap.h

@@ -0,0 +1,99 @@
+/*
+ * hashmap.h - integer hashmap definitions
+ *
+ * Copyright (C) 2019 Emmanuel Marty
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty.  In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ *    claim that you wrote the original software. If you use this software
+ *    in a product, an acknowledgment in the product documentation would be
+ *    appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ *    misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori
+ *
+ * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4
+ * With help, ideas, optimizations and speed measurements by spke <zxintrospec@gmail.com>
+ * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard
+ * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/
+ *
+ */
+
+#ifndef _HASHMAP_H
+#define _HASHMAP_H
+
+#include <stdlib.h>
+
+/** Number of hashmap buckets */
+#define LZSA_HASH_NBUCKETS 256
+
+/* Forward definitions */
+typedef struct _lzsa_hashvalue_t lzsa_hashvalue_t;
+typedef struct _lzsa_hashbuffer_t lzsa_hashbuffer_t;
+
+/** One hashmap bucket entry */
+typedef struct _lzsa_hashvalue_t {
+   lzsa_hashvalue_t *pNext;
+   unsigned long long key;
+   unsigned int value;
+} lzsa_hashvalue_t;
+
+/** One buffer storing hashmap bucket entries */
+typedef struct _lzsa_hashbuffer_t {
+   lzsa_hashbuffer_t *pNext;
+   int nFreeEntryIdx;
+   lzsa_hashvalue_t value[255];
+} lzsa_hashbuffer_t;
+
+/** Hashmap */
+typedef struct {
+   lzsa_hashbuffer_t *pBuffer;
+   lzsa_hashvalue_t *pBucket[LZSA_HASH_NBUCKETS];
+} lzsa_hashmap_t;
+
+/**
+ * Initialize hashmap
+ *
+ * @param pHashMap hashmap
+ */
+void lzsa_hashmap_init(lzsa_hashmap_t *pHashMap);
+
+/**
+ * Set value for key
+ *
+ * @param pHashMap hashmap
+ * @param key key to set value for
+ * @param value new value
+ */
+void lzsa_hashmap_insert(lzsa_hashmap_t *pHashMap, unsigned long long key, unsigned int value);
+
+/**
+ * Get value for key
+ *
+ * @param pHashMap hashmap
+ * @param key key to get value for
+ * @param pValue pointer to where to store value if found
+ *
+ * @return 0 if found, nonzero if not found
+ */
+int lzsa_hashmap_find(lzsa_hashmap_t *pHashMap, unsigned long long key, unsigned int *pValue);
+
+/**
+ * Clear hashmap
+ *
+ * @param pHashMap hashmap
+ */
+void lzsa_hashmap_clear(lzsa_hashmap_t *pHashMap);
+
+#endif   /* _HASHMAP_H */

src/lzsa.c

@@ -46,7 +46,7 @@
 #define OPT_RAW         2
 #define OPT_FAVOR_RATIO 4
 
-#define TOOL_VERSION "1.0.2"
+#define TOOL_VERSION "1.0.3"
 
  /*---------------------------------------------------------------------------*/
 
@@ -127,7 +127,7 @@ static int do_compress(const char *pszInFilename, const char *pszOutFilename, co
       case LZSA_ERROR_MEMORY: fprintf(stderr, "out of memory\n"); break;
       case LZSA_ERROR_COMPRESSION: fprintf(stderr, "internal compression error\n"); break;
       case LZSA_ERROR_RAW_TOOLARGE: fprintf(stderr, "error: raw blocks can only be used with files <= 64 Kb\n"); break;
-      case LZSA_ERROR_RAW_UNCOMPRESSED: fprintf(stderr, "error: data is incompressible, raw blocks only support compressed data\n"); break;
+      case LZSA_ERROR_RAW_UNCOMPRESSED: fprintf(stderr, "error: incompressible data needs to be <= 64 Kb in raw blocks\n"); break;
       case LZSA_OK: break;
       default: fprintf(stderr, "unknown compression error %d\n", nStatus); break;
    }

src/shrink_block_v1.c

@@ -433,6 +433,50 @@ static int lzsa_write_block_v1(lzsa_compressor *pCompressor, const unsigned char
    return nOutOffset;
 }
 
+/**
+ * Emit raw block of uncompressible data
+ *
+ * @param pCompressor compression context
+ * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress)
+ * @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
+ * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
+ * @param pOutData pointer to output buffer
+ * @param nMaxOutDataSize maximum size of output buffer, in bytes
+ *
+ * @return size of compressed data in output buffer, or -1 if the data is uncompressible
+ */
+static int lzsa_write_raw_uncompressed_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) {
+   int nNumLiterals = nEndOffset - nStartOffset;
+   int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V1) ? LITERALS_RUN_LEN_V1 : nNumLiterals;
+   int nOutOffset = 0;
+
+   int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3) + 4;
+   if ((nOutOffset + (nCommandSize >> 3)) > nMaxOutDataSize)
+      return -1;
+
+   pCompressor->num_commands = 0;
+   pOutData[nOutOffset++] = (nTokenLiteralsLen << 4) | 0x0f;
+   
+   nOutOffset = lzsa_write_literals_varlen_v1(pOutData, nOutOffset, nNumLiterals);
+
+   if (nNumLiterals != 0) {
+      memcpy(pOutData + nOutOffset, pInWindow + nStartOffset, nNumLiterals);
+      nOutOffset += nNumLiterals;
+      nNumLiterals = 0;
+   }
+
+   pCompressor->num_commands++;
+
+   /* Emit EOD marker for raw block */
+
+   pOutData[nOutOffset++] = 0;
+   pOutData[nOutOffset++] = 238;
+   pOutData[nOutOffset++] = 0;
+   pOutData[nOutOffset++] = 0;
+
+   return nOutOffset;
+}
+
 /**
  * Select the most optimal matches, reduce the token count if possible, and then emit a block of compressed LZSA1 data
  *
@@ -446,6 +490,8 @@ static int lzsa_write_block_v1(lzsa_compressor *pCompressor, const unsigned char
  * @return size of compressed data in output buffer, or -1 if the data is uncompressible
  */
 int lzsa_optimize_and_write_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) {
+   int nResult;
+
    lzsa_optimize_matches_v1(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
 
    int nDidReduce;
@@ -455,5 +501,10 @@ int lzsa_optimize_and_write_block_v1(lzsa_compressor *pCompressor, const unsigne
       nPasses++;
    } while (nDidReduce && nPasses < 20);
 
-   return lzsa_write_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
+   nResult = lzsa_write_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
+   if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
+      nResult = lzsa_write_raw_uncompressed_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
+   }
+
+   return nResult;
 }

src/shrink_block_v2.c

@@ -35,6 +35,9 @@
 #include "lib.h"
 #include "shrink_block_v2.h"
 #include "format.h"
+#include "hashmap.h"
+
+#define HASH_KEY(__nRepMatchOffset,__nNumLiterals,__i) (((unsigned long long)(__nRepMatchOffset)) | (((unsigned long long)(__i)) << 17) | (((unsigned long long)(__nNumLiterals)) << 34))
 
 /**
  * Write 4-bit nibble to output (compressed) buffer
@@ -222,7 +225,7 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt
       if (pCompressor->best_match[i + 1].length >= MIN_MATCH_SIZE_V2)
          nLiteralsCost += MODESWITCH_PENALTY;
 
-      lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT);
+      const lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT);
       int *pSlotCost = pCompressor->slot_cost + (i << MATCHES_PER_OFFSET_SHIFT);
       int m;
 
@@ -309,8 +312,8 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt
                if (nMatchOffsetSize && nCurIndex < nEndOffset && pCompressor->best_match[nCurIndex].length >= MIN_MATCH_SIZE_V2 && !repmatch_opt[nCurIndex].expected_repmatch) {
                   int r;
 
-                  for (r = 0; r < NMATCHES_PER_OFFSET && pCompressor->match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].length >= MIN_MATCH_SIZE_V2; r++) {
+                  for (r = 0; r < NMATCHES_PER_OFFSET && pCompressor->selected_match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].length >= MIN_MATCH_SIZE_V2; r++) {
-                     if (pCompressor->match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].offset == pMatch[m].offset) {
+                     if (pCompressor->selected_match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].offset == pMatch[m].offset) {
                         int nAltCost = nCurCost - nMatchOffsetSize + pCompressor->slot_cost[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r] - cost[nCurIndex];
 
                         if (nAltCost <= nCurCost) {
@@ -356,8 +359,8 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt
                if (nMatchOffsetSize && nCurIndex < nEndOffset && pCompressor->best_match[nCurIndex].length >= MIN_MATCH_SIZE_V2 && !repmatch_opt[nCurIndex].expected_repmatch) {
                   int r;
 
-                  for (r = 0; r < NMATCHES_PER_OFFSET && pCompressor->match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].length >= MIN_MATCH_SIZE_V2; r++) {
+                  for (r = 0; r < NMATCHES_PER_OFFSET && pCompressor->selected_match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].length >= MIN_MATCH_SIZE_V2; r++) {
-                     if (pCompressor->match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].offset == pMatch[m].offset) {
+                     if (pCompressor->selected_match[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r].offset == pMatch[m].offset) {
                         int nAltCost = nCurCost - nMatchOffsetSize + pCompressor->slot_cost[(nCurIndex << MATCHES_PER_OFFSET_SHIFT) + r] - cost[nCurIndex];
 
                         if (nAltCost <= nCurCost) {
@@ -382,8 +385,8 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt
          }
 
          pSlotCost[m] = nBestCost;         
-         pMatch[m].length = nBestMatchLen;
+         pCompressor->selected_match[(i << MATCHES_PER_OFFSET_SHIFT) + m].length = nBestMatchLen;
-         pMatch[m].offset = nBestMatchOffset; /* not necessary */
+         pCompressor->selected_match[(i << MATCHES_PER_OFFSET_SHIFT) + m].offset = nBestMatchOffset;
 
          if (m == 0 || (nBestMatchLen && cost[i] >= nBestCost)) {
             cost[i] = nBestCost;
@@ -400,6 +403,7 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt
       }
       for (; m < NMATCHES_PER_OFFSET; m++) {
          pSlotCost[m] = 0;
+         pCompressor->selected_match[(i << MATCHES_PER_OFFSET_SHIFT) + m] = pMatch[m];
       }
 
       if (pCompressor->best_match[i].length >= MIN_MATCH_SIZE_V2)
@@ -412,7 +416,7 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt
    for (i = nStartOffset; i < nEndOffset; ) {
       if (pCompressor->best_match[i].length >= MIN_MATCH_SIZE_V2) {
          if (nIncomingOffset >= 0 && repmatch_opt[i].incoming_offset == nIncomingOffset && repmatch_opt[i].best_slot_for_incoming >= 0) {
-            lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT) + repmatch_opt[i].best_slot_for_incoming;
+            lzsa_match *pMatch = pCompressor->selected_match + (i << MATCHES_PER_OFFSET_SHIFT) + repmatch_opt[i].best_slot_for_incoming;
             int *pSlotCost = pCompressor->slot_cost + (i << MATCHES_PER_OFFSET_SHIFT) + repmatch_opt[i].best_slot_for_incoming;
 
             pCompressor->best_match[i].length = pMatch->length;
@@ -446,7 +450,7 @@ static void lzsa_optimize_matches_v2(lzsa_compressor *pCompressor, const int nSt
  *
  * @return non-zero if the number of tokens was reduced, 0 if it wasn't
  */
-static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) {
+static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) {
    int i;
    int nNumLiterals = 0;
    int nDidReduce = 0;
@@ -455,7 +459,7 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in
    lzsa_repmatch_opt *repmatch_opt = pCompressor->repmatch_opt;
 
    for (i = nStartOffset; i < nEndOffset; ) {
-      lzsa_match *pMatch = pCompressor->best_match + i;
+      lzsa_match *pMatch = pBestMatch + i;
 
       if (pMatch->length >= MIN_MATCH_SIZE_V2) {
          int nMatchLen = pMatch->length;
@@ -468,10 +472,10 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in
             int nRepMatchSize = (nRepMatchOffset <= 32) ? 4 : ((nRepMatchOffset <= 512) ? 8 : ((nRepMatchOffset <= (8192 + 512)) ? 12 : 16)) /* match offset */;
             int nUndoRepMatchCost = (nPreviousMatchOffset < 0 || !repmatch_opt[nPreviousMatchOffset].expected_repmatch) ? 0 : nRepMatchSize;
 
-            if (pCompressor->best_match[i + nMatchLen].length >= MIN_MATCH_SIZE_V2) {
+            if (pBestMatch[i + nMatchLen].length >= MIN_MATCH_SIZE_V2) {
                int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + lzsa_get_match_varlen_size_v2(nEncodedMatchLen) - nUndoRepMatchCost;
 
-               if (pCompressor->best_match[i + nMatchLen].offset != nMatchOffset) {
+               if (pBestMatch[i + nMatchLen].offset != nMatchOffset) {
                   nCommandSize += (nMatchOffset <= 32) ? 4 : ((nMatchOffset <= 512) ? 8 : ((nMatchOffset <= (8192 + 512)) ? 12 : 16)) /* match offset */;
                }
 
@@ -484,7 +488,7 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in
                }
                else {
                   if (nMatchOffset != nRepMatchOffset &&
-                      pCompressor->best_match[i + nMatchLen].offset == nRepMatchOffset) {
+                      pBestMatch[i + nMatchLen].offset == nRepMatchOffset) {
 
                      if (nCommandSize > ((nMatchLen << 3) + lzsa_get_literals_varlen_size_v2(nNumLiterals + nMatchLen) - nRepMatchSize)) {
                         /* Same case, replacing this command by literals alone isn't enough on its own to have savings, however this match command is inbetween two matches with
@@ -503,10 +507,10 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in
                do {
                   nCurIndex++;
                   nNextNumLiterals++;
-               } while (nCurIndex < nEndOffset && pCompressor->best_match[nCurIndex].length < MIN_MATCH_SIZE_V2);
+               } while (nCurIndex < nEndOffset && pBestMatch[nCurIndex].length < MIN_MATCH_SIZE_V2);
 
-               if (nCurIndex >= nEndOffset || pCompressor->best_match[nCurIndex].length < MIN_MATCH_SIZE_V2 ||
+               if (nCurIndex >= nEndOffset || pBestMatch[nCurIndex].length < MIN_MATCH_SIZE_V2 ||
-                  pCompressor->best_match[nCurIndex].offset != nMatchOffset) {
+                  pBestMatch[nCurIndex].offset != nMatchOffset) {
                   nCommandSize += (nMatchOffset <= 32) ? 4 : ((nMatchOffset <= 512) ? 8 : ((nMatchOffset <= (8192 + 512)) ? 12 : 16)) /* match offset */;
                }
 
@@ -516,9 +520,9 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in
                   nReduce = 1;
                }
                else {
-                  if (nCurIndex < nEndOffset && pCompressor->best_match[nCurIndex].length >= MIN_MATCH_SIZE_V2 &&
+                  if (nCurIndex < nEndOffset && pBestMatch[nCurIndex].length >= MIN_MATCH_SIZE_V2 &&
-                     pCompressor->best_match[nCurIndex].offset != nMatchOffset &&
+                     pBestMatch[nCurIndex].offset != nMatchOffset &&
-                     pCompressor->best_match[nCurIndex].offset == nRepMatchOffset) {
+                     pBestMatch[nCurIndex].offset == nRepMatchOffset) {
                      if (nCommandSize > ((nMatchLen << 3) + lzsa_get_literals_varlen_size_v2(nNumLiterals + nNextNumLiterals + nMatchLen) - lzsa_get_literals_varlen_size_v2(nNextNumLiterals) - nRepMatchSize)) {
                         /* Same case, but now replacing this command allows to use a rep-match and get savings, so do it */
                         nReduce = 1;
@@ -532,7 +536,7 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in
             int j;
 
             for (j = 0; j < nMatchLen; j++) {
-               pCompressor->best_match[i + j].length = 0;
+               pBestMatch[i + j].length = 0;
             }
             nNumLiterals += nMatchLen;
             i += nMatchLen;
@@ -549,13 +553,13 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in
                nRepMatchOffset = pMatch->offset;
 
             if ((i + nMatchLen) < nEndOffset && nMatchLen >= LCP_MAX &&
-               pMatch->offset && pMatch->offset <= 32 && pCompressor->best_match[i + nMatchLen].offset == pMatch->offset && (nMatchLen % pMatch->offset) == 0 &&
+               pMatch->offset && pMatch->offset <= 32 && pBestMatch[i + nMatchLen].offset == pMatch->offset && (nMatchLen % pMatch->offset) == 0 &&
-               (nMatchLen + pCompressor->best_match[i + nMatchLen].length) <= MAX_VARLEN) {
+               (nMatchLen + pBestMatch[i + nMatchLen].length) <= MAX_VARLEN) {
                /* Join */
 
-               pMatch->length += pCompressor->best_match[i + nMatchLen].length;
+               pMatch->length += pBestMatch[i + nMatchLen].length;
-               pCompressor->best_match[i + nMatchLen].offset = 0;
+               pBestMatch[i + nMatchLen].offset = 0;
-               pCompressor->best_match[i + nMatchLen].length = -1;
+               pBestMatch[i + nMatchLen].length = -1;
                continue;
             }
 
@@ -574,10 +578,252 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in
    return nDidReduce;
 }
 
+/**
+ * Get cost of the best encoding choice at a given offset, going forward
+ *
+ * @param pCompressor compression context
+ * @param i offset in input window
+ * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes)
+ * @param nNumLiterals current pending number of literals to be encoded with the next token
+ * @param nRepMatchOffset current rep-match offset
+ * @param nDepth current recursion depth
+ * @param pBestMatchLen pointer to returned best match length for the position (0 for no match)
+ * @param pBestMatchOffset pointer to returned best match offset for the position (if there is a match)
+ *
+ * @return cost of best encoding choice for offset
+ */
+static int lzsa_get_forward_cost_v2(lzsa_compressor *pCompressor, const int i, const int nEndOffset, const int nNumLiterals, const int nRepMatchOffset, int nDepth, int *pBestMatchLen, int *pBestMatchOffset) {
+   if (i >= nEndOffset)
+      return 0;
+
+   int *cost = (int*)pCompressor->pos_data;  /* Reuse */
+   if (nDepth >= pCompressor->max_forward_depth)
+      return cost[i];
+
+   if (nDepth >= 1) {
+      unsigned int nValue = 0;
+
+      if (!lzsa_hashmap_find(&pCompressor->cost_map, HASH_KEY(nRepMatchOffset, nNumLiterals, i), &nValue))
+         return nValue;
+   }
+
+   int nMinMatchSize = pCompressor->min_match_size;
+   int m;
+   const lzsa_match *pMatch = pCompressor->match + (i << MATCHES_PER_OFFSET_SHIFT);
+
+   int nBestCost, nBestMatchLen, nBestMatchOffset, nTmpMatchLen, nTmpMatchOffset;
+   int nLiteralsCost;
+
+   nBestCost = 8 + lzsa_get_forward_cost_v2(pCompressor, i + 1, nEndOffset, nNumLiterals + 1, nRepMatchOffset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset);
+   nBestMatchLen = 0;
+   nBestMatchOffset = 0;
+
+   nLiteralsCost = lzsa_get_literals_varlen_size_v2(nNumLiterals);
+
+   for (m = 0; m < NMATCHES_PER_OFFSET && pMatch[m].length >= nMinMatchSize; m++) {
+      if (pMatch[m].length > 30) {
+         int nCurCost;
+         int nMatchLen = pMatch[m].length;
+
+         if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS))
+            nMatchLen = nEndOffset - LAST_LITERALS - i;
+
+         int nMatchOffsetSize;
+         if (nRepMatchOffset == pMatch[m].offset)
+            nMatchOffsetSize = 0;
+         else {
+            nMatchOffsetSize = (pMatch[m].offset <= 32) ? 4 : ((pMatch[m].offset <= 512) ? 8 : ((pMatch[m].offset <= (8192 + 512)) ? 12 : 16));
+         }
+
+         nCurCost = 8 + nLiteralsCost + nMatchOffsetSize + lzsa_get_match_varlen_size_v2(nMatchLen - MIN_MATCH_SIZE_V2);
+         nCurCost += lzsa_get_forward_cost_v2(pCompressor, i + nMatchLen, nEndOffset, 0, pMatch[m].offset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset);
+
+         if (nBestCost >= nCurCost) {
+            nBestCost = nCurCost;
+            nBestMatchLen = nMatchLen;
+            nBestMatchOffset = pMatch[m].offset;
+         }
+      }
+      else {
+         int nMatchLen = pMatch[m].length;
+         int k, nMatchRunLen;
+
+         if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS))
+            nMatchLen = nEndOffset - LAST_LITERALS - i;
+
+         nMatchRunLen = nMatchLen;
+         if (nMatchRunLen > MATCH_RUN_LEN_V2)
+            nMatchRunLen = MATCH_RUN_LEN_V2;
+
+         for (k = nMinMatchSize; k < nMatchRunLen; k++) {
+            int nCurCost;
+
+            int nMatchOffsetSize;
+            if (nRepMatchOffset == pMatch[m].offset)
+               nMatchOffsetSize = 0;
+            else {
+               nMatchOffsetSize = (pMatch[m].offset <= 32) ? 4 : ((pMatch[m].offset <= 512) ? 8 : ((pMatch[m].offset <= (8192 + 512)) ? 12 : 16));
+            }
+
+            nCurCost = 8 + nLiteralsCost + nMatchOffsetSize /* no extra match len bytes */;
+            nCurCost += lzsa_get_forward_cost_v2(pCompressor, i + k, nEndOffset, 0, pMatch[m].offset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset);
+
+            if (nBestCost >= nCurCost) {
+               nBestCost = nCurCost;
+               nBestMatchLen = k;
+               nBestMatchOffset = pMatch[m].offset;
+            }
+         }
+
+         for (; k <= nMatchLen; k++) {
+            int nCurCost;
+
+            int nMatchOffsetSize;
+            if (nRepMatchOffset == pMatch[m].offset)
+               nMatchOffsetSize = 0;
+            else {
+               nMatchOffsetSize = (pMatch[m].offset <= 32) ? 4 : ((pMatch[m].offset <= 512) ? 8 : ((pMatch[m].offset <= (8192 + 512)) ? 12 : 16));
+            }
+
+            nCurCost = 8 + nLiteralsCost + nMatchOffsetSize + lzsa_get_match_varlen_size_v2(k - MIN_MATCH_SIZE_V2);
+            nCurCost += lzsa_get_forward_cost_v2(pCompressor, i + k, nEndOffset, 0, pMatch[m].offset, nDepth + 1, &nTmpMatchLen, &nTmpMatchOffset);
+
+            if (nBestCost >= nCurCost) {
+               nBestCost = nCurCost;
+               nBestMatchLen = k;
+               nBestMatchOffset = pMatch[m].offset;
+            }
+         }
+      }
+   }
+
+   *pBestMatchLen = nBestMatchLen;
+   *pBestMatchOffset = nBestMatchOffset;
+
+   lzsa_hashmap_insert(&pCompressor->cost_map, HASH_KEY(nRepMatchOffset, nNumLiterals, i), nBestCost);
+   return nBestCost;
+}
+
+/**
+ * Attempt to further improve the selected optimal matches with a chain-N forward parser pass
+ *
+ * @param pCompressor compression context
+ * @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
+ * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
+ */
+static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) {
+   int i;
+   int nNumLiterals = 0;
+   int nRepMatchOffset = 0;
+   int *cost = (int*)pCompressor->pos_data;  /* Reuse */
+
+   lzsa_hashmap_init(&pCompressor->cost_map);
+
+   for (i = nStartOffset; i < nEndOffset; ) {
+      int nBestMatchLen = 0, nBestMatchOffset = 0;
+      int nBestCost = lzsa_get_forward_cost_v2(pCompressor, i, nEndOffset, nNumLiterals, nRepMatchOffset, 0, &nBestMatchLen, &nBestMatchOffset);
+
+      lzsa_hashmap_clear(&pCompressor->cost_map);
+
+      lzsa_match *pMatch = pCompressor->improved_match + i;
+      if (nBestCost < cost[i]) {
+         pMatch->length = nBestMatchLen;
+         pMatch->offset = nBestMatchOffset;
+      }
+
+      if (pMatch->length >= MIN_MATCH_SIZE_V2) {
+         nNumLiterals = 0;
+         nRepMatchOffset = pMatch->offset;
+
+         i += pMatch->length;
+      }
+      else {
+         nNumLiterals++;
+         i++;
+      }
+   }
+
+   lzsa_hashmap_clear(&pCompressor->cost_map);
+}
+
+/**
+ * Calculate compressed size
+ *
+ * @param pCompressor compression context
+ * @param pBestMatch optimal matches to evaluate
+ * @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
+ * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
+ *
+ * @return compressed size
+ */
+static int lzsa_get_compressed_size_v2(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) {
+   int i;
+   int nNumLiterals = 0;
+   int nCompressedSize = 0;
+   int nCurNibbleOffset = -1, nCurFreeNibbles = 0;
+   int nRepMatchOffset = 0;
+
+   for (i = nStartOffset; i < nEndOffset; ) {
+      const lzsa_match *pMatch = pBestMatch + i;
+
+      if (pMatch->length >= MIN_MATCH_SIZE_V2) {
+         int nMatchOffset = pMatch->offset;
+         int nMatchLen = pMatch->length;
+         int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE_V2;
+         int nOffsetSize;
+
+         if (nMatchOffset == nRepMatchOffset) {
+            nOffsetSize = 0;
+         }
+         else {
+            if (nMatchOffset <= 32) {
+               nOffsetSize = 4;
+            }
+            else if (nMatchOffset <= 512) {
+               nOffsetSize = 8;
+            }
+            else if (nMatchOffset <= (8192 + 512)) {
+               nOffsetSize = 12;
+            }
+            else {
+               nOffsetSize = 16;
+            }
+         }
+
+         int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3) + nOffsetSize /* match offset */ + lzsa_get_match_varlen_size_v2(nEncodedMatchLen);
+         nCompressedSize += nCommandSize;
+
+         nNumLiterals = 0;
+         nRepMatchOffset = nMatchOffset;
+
+         i += nMatchLen;
+      }
+      else {
+         nNumLiterals++;
+         i++;
+      }
+   }
+
+   {
+      int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V2) ? LITERALS_RUN_LEN_V2 : nNumLiterals;
+      int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3);
+
+      nCompressedSize += nCommandSize;
+      nNumLiterals = 0;
+   }
+
+   if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
+      nCompressedSize += 8 + 4 + 8;
+   }
+
+   return nCompressedSize;
+}
+
 /**
  * Emit block of compressed data
  *
  * @param pCompressor compression context
+ * @param pBestMatch optimal matches to emit
  * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress)
  * @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
  * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
@@ -586,7 +832,7 @@ static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const in
  *
  * @return size of compressed data in output buffer, or -1 if the data is uncompressible
  */
-static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) {
+static int lzsa_write_block_v2(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) {
    int i;
    int nNumLiterals = 0;
    int nInFirstLiteralOffset = 0;
@@ -595,7 +841,7 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const unsigned char
    int nRepMatchOffset = 0;
 
    for (i = nStartOffset; i < nEndOffset; ) {
-      lzsa_match *pMatch = pCompressor->best_match + i;
+      const lzsa_match *pMatch = pBestMatch + i;
 
       if (pMatch->length >= MIN_MATCH_SIZE_V2) {
          int nMatchOffset = pMatch->offset;
@@ -727,6 +973,63 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const unsigned char
    return nOutOffset;
 }
 
+/**
+ * Emit raw block of uncompressible data
+ *
+ * @param pCompressor compression context
+ * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress)
+ * @param nStartOffset current offset in input window (typically the number of previously compressed bytes)
+ * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes
+ * @param pOutData pointer to output buffer
+ * @param nMaxOutDataSize maximum size of output buffer, in bytes
+ *
+ * @return size of compressed data in output buffer, or -1 if the data is uncompressible
+ */
+static int lzsa_write_raw_uncompressed_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) {
+   int nCurNibbleOffset = -1, nCurFreeNibbles = 0;
+   int nNumLiterals = nEndOffset - nStartOffset;
+   int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V2) ? LITERALS_RUN_LEN_V2 : nNumLiterals;
+   int nOutOffset = 0;
+
+   int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3) + 8 + 4 + 8;
+   if ((nOutOffset + ((nCommandSize + 7) >> 3)) > nMaxOutDataSize)
+      return -1;
+
+   pCompressor->num_commands = 0;
+   pOutData[nOutOffset++] = (nTokenLiteralsLen << 3) | 0x47;
+
+   nOutOffset = lzsa_write_literals_varlen_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, &nCurFreeNibbles, nNumLiterals);
+   if (nOutOffset < 0) return -1;
+
+   if (nNumLiterals != 0) {
+      memcpy(pOutData + nOutOffset, pInWindow + nStartOffset, nNumLiterals);
+      nOutOffset += nNumLiterals;
+      nNumLiterals = 0;
+   }
+
+   /* Emit EOD marker for raw block */
+
+   pOutData[nOutOffset++] = 0;      /* Match offset */
+
+   nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, &nCurFreeNibbles, 15);   /* Extended match length nibble */
+   if (nOutOffset < 0) return -1;
+
+   if ((nOutOffset + 1) > nMaxOutDataSize)
+      return -1;
+
+   pOutData[nOutOffset++] = 232;    /* EOD match length byte */
+
+   pCompressor->num_commands++;
+
+   if (nCurNibbleOffset != -1) {
+      nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, &nCurFreeNibbles, 0);
+      if (nOutOffset < 0 || nCurNibbleOffset != -1)
+         return -1;
+   }
+
+   return nOutOffset;
+}
+
 /**
  * Select the most optimal matches, reduce the token count if possible, and then emit a block of compressed LZSA2 data
  *
@@ -740,14 +1043,49 @@ static int lzsa_write_block_v2(lzsa_compressor *pCompressor, const unsigned char
  * @return size of compressed data in output buffer, or -1 if the data is uncompressible
  */
 int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) {
+   int nResult;
+
    lzsa_optimize_matches_v2(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
+   lzsa_match *pBestMatch;
+   if (pCompressor->max_forward_depth > 0) {
+      memcpy(pCompressor->improved_match, pCompressor->best_match, nInDataSize * sizeof(lzsa_match));
+      lzsa_optimize_forward_v2(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
 
-   int nDidReduce;
+      int nDidReduce;
-   int nPasses = 0;
+      int nPasses = 0;
-   do {
+      do {
-      nDidReduce = lzsa_optimize_command_count_v2(pCompressor, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
+         nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
-      nPasses++;
+         nPasses++;
-   } while (nDidReduce && nPasses < 20);
+      } while (nDidReduce && nPasses < 20);
 
-   return lzsa_write_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
+      nPasses = 0;
+      do {
+         nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pCompressor->improved_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
+         nPasses++;
+      } while (nDidReduce && nPasses < 20);
+
+      int nBestCost = lzsa_get_compressed_size_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
+      int nImprovedCost = lzsa_get_compressed_size_v2(pCompressor, pCompressor->improved_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
+      if (nBestCost > nImprovedCost)
+         pBestMatch = pCompressor->improved_match;
+      else
+         pBestMatch = pCompressor->best_match;
+   }
+   else {
+      int nDidReduce;
+      int nPasses = 0;
+      do {
+         nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pCompressor->best_match, nPreviousBlockSize, nPreviousBlockSize + nInDataSize);
+         nPasses++;
+      } while (nDidReduce && nPasses < 20);
+
+      pBestMatch = pCompressor->best_match;
+   }
+
+   nResult = lzsa_write_block_v2(pCompressor, pBestMatch, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
+   if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) {
+      nResult = lzsa_write_raw_uncompressed_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize);
+   }
+
+   return nResult;
 }

src/shrink_context.c

@@ -58,7 +58,9 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
    pCompressor->pos_data = NULL;
    pCompressor->open_intervals = NULL;
    pCompressor->match = NULL;
+   pCompressor->selected_match = NULL;
    pCompressor->best_match = NULL;
+   pCompressor->improved_match = NULL;
    pCompressor->slot_cost = NULL;
    pCompressor->repmatch_opt = NULL;
    pCompressor->min_match_size = nMinMatchSize;
@@ -66,6 +68,7 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
       pCompressor->min_match_size = nMinMatchSizeForFormat;
    else if (pCompressor->min_match_size > nMaxMinMatchForFormat)
       pCompressor->min_match_size = nMaxMinMatchForFormat;
+   pCompressor->max_forward_depth = 0;
    pCompressor->format_version = nFormatVersion;
    pCompressor->flags = nFlags;
    pCompressor->num_commands = 0;
@@ -84,16 +87,24 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
 
                if (pCompressor->match) {
                   if (pCompressor->format_version == 2) {
-                     pCompressor->best_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match));
+                     pCompressor->selected_match = (lzsa_match *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(lzsa_match));
 
-                     if (pCompressor->best_match) {
+                     if (pCompressor->selected_match) {
-                        pCompressor->slot_cost = (int *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(int));
+                        pCompressor->best_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match));
 
-                        if (pCompressor->slot_cost) {
+                        if (pCompressor->best_match) {
-                           pCompressor->repmatch_opt = (lzsa_repmatch_opt *)malloc(nMaxWindowSize * sizeof(lzsa_repmatch_opt));
+                           pCompressor->improved_match = (lzsa_match *)malloc(nMaxWindowSize * sizeof(lzsa_match));
 
-                           if (pCompressor->repmatch_opt)
+                           if (pCompressor->improved_match) {
-                              return 0;
+                              pCompressor->slot_cost = (int *)malloc(nMaxWindowSize * NMATCHES_PER_OFFSET * sizeof(int));
+
+                              if (pCompressor->slot_cost) {
+                                 pCompressor->repmatch_opt = (lzsa_repmatch_opt *)malloc(nMaxWindowSize * sizeof(lzsa_repmatch_opt));
+
+                                 if (pCompressor->repmatch_opt)
+                                    return 0;
+                              }
+                           }
                         }
                      }
                   }
@@ -128,11 +139,21 @@ void lzsa_compressor_destroy(lzsa_compressor *pCompressor) {
       pCompressor->slot_cost = NULL;
    }
 
+   if (pCompressor->improved_match) {
+      free(pCompressor->improved_match);
+      pCompressor->improved_match = NULL;
+   }
+
    if (pCompressor->best_match) {
       free(pCompressor->best_match);
       pCompressor->best_match = NULL;
    }
 
+   if (pCompressor->selected_match) {
+      free(pCompressor->selected_match);
+      pCompressor->selected_match = NULL;
+   }
+
    if (pCompressor->match) {
       free(pCompressor->match);
       pCompressor->match = NULL;

src/shrink_context.h

@@ -34,6 +34,7 @@
 #define _SHRINK_CONTEXT_H
 
 #include "divsufsort.h"
+#include "hashmap.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -77,13 +78,17 @@ typedef struct _lzsa_compressor {
    unsigned int *pos_data;
    unsigned int *open_intervals;
    lzsa_match *match;
+   lzsa_match *selected_match;
    lzsa_match *best_match;
+   lzsa_match *improved_match;
    int *slot_cost;
    lzsa_repmatch_opt *repmatch_opt;
    int min_match_size;
+   int max_forward_depth;
    int format_version;
    int flags;
    int num_commands;
+   lzsa_hashmap_t cost_map;
 } lzsa_compressor;
 
 /**

src/shrink_inmem.c

@@ -84,6 +84,21 @@ size_t lzsa_compress_inmem(const unsigned char *pInputData, unsigned char *pOutB
       }
    }
 
+   if ((compressor.flags & LZSA_FLAG_FAVOR_RATIO)) {
+      if (nInputSize < 16384)
+         compressor.max_forward_depth = 25;
+      else {
+         if (nInputSize < 32768)
+            compressor.max_forward_depth = 15;
+         else {
+            if (nInputSize < BLOCK_SIZE)
+               compressor.max_forward_depth = 10;
+            else
+               compressor.max_forward_depth = 0;
+         }
+      }
+   }
+
    int nPreviousBlockSize = 0;
    int nNumBlocks = 0;
 

src/shrink_streaming.c

@@ -36,6 +36,24 @@
 #include "format.h"
 #include "frame.h"
 #include "lib.h"
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <stdio.h>
+#endif
+
+/**
+ * Delete file
+ *
+ * @param pszInFilename name of file to delete
+ */
+static void lzsa_delete_file(const char *pszInFilename) {
+#ifdef _WIN32
+   DeleteFileA(pszInFilename);
+#else
+   remove(pszInFilename);
+#endif
+}
 
 /*-------------- File API -------------- */
 
@@ -76,6 +94,7 @@ lzsa_status_t lzsa_compress_file(const char *pszInFilename, const char *pszOutFi
    if (nStatus) {
       outStream.close(&outStream);
       inStream.close(&inStream);
+      lzsa_delete_file(pszOutFilename);
       return nStatus;
    }
 
@@ -84,6 +103,11 @@ lzsa_status_t lzsa_compress_file(const char *pszInFilename, const char *pszOutFi
    lzsa_dictionary_free(&pDictionaryData);
    outStream.close(&outStream);
    inStream.close(&inStream);
+
+   if (nStatus) {
+      lzsa_delete_file(pszOutFilename);
+   }
+
    return nStatus;
 }
 
@@ -115,6 +139,7 @@ lzsa_status_t lzsa_compress_stream(lzsa_stream_t *pInStream, lzsa_stream_t *pOut
    int nResult;
    unsigned char cFrameData[16];
    int nError = 0;
+   int nRawPadding = (nFlags & LZSA_FLAG_RAW_BLOCK) ? 8 : 0;
 
    pInData = (unsigned char*)malloc(BLOCK_SIZE * 2);
    if (!pInData) {
@@ -175,9 +200,24 @@ lzsa_status_t lzsa_compress_stream(lzsa_stream_t *pInStream, lzsa_stream_t *pOut
          }
          nDictionaryDataSize = 0;
 
+         if (nNumBlocks == 0 && (compressor.flags & LZSA_FLAG_FAVOR_RATIO)) {
+            if (nInDataSize < 16384)
+               compressor.max_forward_depth = 25;
+            else {
+               if (nInDataSize < 32768)
+                  compressor.max_forward_depth = 15;
+               else {
+                  if (nInDataSize < BLOCK_SIZE)
+                     compressor.max_forward_depth = 10;
+                  else
+                     compressor.max_forward_depth = 0;
+               }
+            }
+         }
+
          int nOutDataSize;
 
-         nOutDataSize = lzsa_compressor_shrink_block(&compressor, pInData + BLOCK_SIZE - nPreviousBlockSize, nPreviousBlockSize, nInDataSize, pOutData, (nInDataSize >= BLOCK_SIZE) ? BLOCK_SIZE : nInDataSize);
+         nOutDataSize = lzsa_compressor_shrink_block(&compressor, pInData + BLOCK_SIZE - nPreviousBlockSize, nPreviousBlockSize, nInDataSize, pOutData, ((nInDataSize + nRawPadding) >= BLOCK_SIZE) ? BLOCK_SIZE : (nInDataSize + nRawPadding));
          if (nOutDataSize >= 0) {
             /* Write compressed block */