6502/lzsa
1
0
Fork 0
mirror of https://github.com/emmanuel-marty/lzsa.git synced 2025-02-05 21:35:30 +00:00
Code Issues Projects Releases Wiki Activity

Revert token to O|LLL|MMMM; revert to always shifting the match offset by 1; set raw block end marker as a large zero-size match

Browse Source
This commit is contained in:
emmanuel-marty 2019-04-05 23:16:05 +02:00
parent 06e6a14871
commit a785010448
5 changed files with 56 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@ -75,7 +75,7 @@ Each frame contains a 3-bytes length followed by block data that expands to up t
LZSA blocks are composed from consecutive commands. Each command follows this format: LZSA blocks are composed from consecutive commands. Each command follows this format:
* token: <LLL|MMMM|O> * token: <O|LLL|MMMM>
* optional extra literal length * optional extra literal length
* literal values * literal values
* match offset low * match offset low
@ -87,7 +87,7 @@ LZSA blocks are composed from consecutive commands. Each command follows this fo
The token byte is broken down into three parts: The token byte is broken down into three parts:
7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0
L L L M M M M O O L L L M M M M
* L: 3-bit literals length (0-6, or 7 if extended). If the number of literals for this command is 0 to 6, the length is encoded in the token and no extra bytes are required. Otherwise, a value of 7 is encoded and extra bytes follow as 'optional extra literal length' * L: 3-bit literals length (0-6, or 7 if extended). If the number of literals for this command is 0 to 6, the length is encoded in the token and no extra bytes are required. Otherwise, a value of 7 is encoded and extra bytes follow as 'optional extra literal length'
* M: 4-bit encoded match length (0-14, or 15 if extended). Likewise, if the encoded match length for this command is 0 to 14, it is directly stored, otherwise 15 is stored and extra bytes follow as 'optional extra encoded match length'. Except for the last command in a block, a command always contains a match, so the encoded match length is the actual match length offset by the minimum, which is 3 bytes. For instance, an actual match length of 10 bytes to be copied, is encoded as 7. * M: 4-bit encoded match length (0-14, or 15 if extended). Likewise, if the encoded match length for this command is 0 to 14, it is directly stored, otherwise 15 is stored and extra bytes follow as 'optional extra encoded match length'. Except for the last command in a block, a command always contains a match, so the encoded match length is the actual match length offset by the minimum, which is 3 bytes. For instance, an actual match length of 10 bytes to be copied, is encoded as 7.
@ -113,21 +113,19 @@ The low 8 bits of the match offset follows.
**optional match offset high** **optional match offset high**
If the 'O' bit (bit 0) is set in the token, the high 8 bits of the match offset follow, otherwise they are understood to be all set to 0. If the 'O' bit (bit 7) is set in the token, the high 8 bits of the match offset follow, otherwise they are understood to be all set to 0.
**important note regarding short match offsets: off by 1** **important note regarding match offsets: off by 1**
Note that the match offset is *off by 1* when encoded as a single byte (the O bit in the token is set to 0): a value of 0 refers to the byte preceding the current output index (N-1). A value of 1 refers to two bytes before the current output index (N-2) and so on. This is so that match offsets up to 256 can be encoded as a single byte, for extra compression. Note that the match offset is *off by 1*: a value of 0 refers to the byte preceding the current output index (N-1). A value of 1 refers to two bytes before the current output index (N-2) and so on. This is so that match offsets up to 256 can be encoded as a single byte, for extra compression.
When match offsets are encoded as two bytes (the O bit in the token is set to 1), they are stored directly: a value of 1 refers to the byte preceding the current output index (N-1), and so on.
**optional extra encoded match length** **optional extra encoded match length**
If the encoded match length is 15 or more, the 'M' bits in the token form the value 15, and an extra byte follows here, with three possible types of value. If the encoded match length is 15 or more, the 'M' bits in the token form the value 15, and an extra byte follows here, with three possible types of value.
* 0-253: the value is added to the 15 stored in the token. * 0-253: the value is added to the 15 stored in the token. The final value is 3 + 15 + this byte.
* 254: a second byte follows. The final encoded match length is 15 + 254 + the second byte, which gives an actual match length of 3 + 15 + 254 + the second byte. * 254: a second byte follows. The final encoded match length is 15 + 254 + the second byte, which gives an actual match length of 3 + 15 + 254 + the second byte.
* 255: a second and third byte follow, forming a little-endian 16-bit value. The final encoded match length is that 16-bit value. * 255: a second and third byte follow, forming a little-endian 16-bit value. The final encoded match length is 3 + that 16-bit value.
# Footer format # Footer format

17
asm/8088/decompress_small.S
View File

@ -41,9 +41,9 @@ lzsa_decompress:
lodsb ; read token byte: LLL|MMMM|O lodsb ; read token byte: LLL|MMMM|O
mov dx,ax ; keep token in dl mov dx,ax ; keep token in dl
mov cl,3 and al,070H ; isolate literals length in token (LLL)
rol al,cl ; shift literals length into place mov cl,4
and al,07H ; isolate literals length in token (LLL) shr al,cl ; shift literals length into place
mov cx,ax ; copy literals length into cx mov cx,ax ; copy literals length into cx
cmp al,07H ; LITERALS_RUN_LEN? cmp al,07H ; LITERALS_RUN_LEN?
@ -54,20 +54,18 @@ lzsa_decompress:
.copy_literals: .copy_literals:
rep movsb ; copy cx literals from ds:si to es:di rep movsb ; copy cx literals from ds:si to es:di
ror dl,1 ; check match offset size in token (O bit) test dl,dl ; check match offset size in token (O bit)
jc .get_long_offset js .get_long_offset
xchg ax,cx ; clear ah - cx is zero from the rep movsb above xchg ax,cx ; clear ah - cx is zero from the rep movsb above
lodsb lodsb
inc ax ; the match offset is stored off-by-1, increase it
jmp short .get_match_length jmp short .get_match_length
.get_long_offset: .get_long_offset:
lodsw ; Get 2-byte match offset lodsw ; Get 2-byte match offset
test ax,ax
je short .done_decompressing ; bail if we hit EOD
.get_match_length: .get_match_length:
inc ax ; the match offset is stored off-by-1, increase it
xchg ax,dx ; dx: match offset ax: original token xchg ax,dx ; dx: match offset ax: original token
and al,0FH ; isolate match length in token (MMMM) and al,0FH ; isolate match length in token (MMMM)
@ -76,6 +74,8 @@ lzsa_decompress:
jne .copy_match ; no, we have the full match length from the token, go copy jne .copy_match ; no, we have the full match length from the token, go copy
call .get_varlen ; get complete match length call .get_varlen ; get complete match length
test cx,cx
je short .done_decompressing ; bail if we hit EOD
.copy_match: .copy_match:
add cx,3 ; add MIN_MATCH_SIZE to get the final match length to copy add cx,3 ; add MIN_MATCH_SIZE to get the final match length to copy
@ -113,4 +113,5 @@ lzsa_decompress:
.large_varlen: .large_varlen:
lodsw ; grab 16-bit extra length lodsw ; grab 16-bit extra length
mov cx,ax
ret ret

22
src/expand.c
View File

@ -168,7 +168,7 @@ int lzsa_expand_block(const unsigned char *pInBlock, int nBlockSize, unsigned ch
while (pInBlock < pInBlockFastEnd && pCurOutData < pOutDataFastEnd) { while (pInBlock < pInBlockFastEnd && pCurOutData < pOutDataFastEnd) {
const unsigned char token = *pInBlock++; const unsigned char token = *pInBlock++;
int nLiterals = (int)((unsigned int)((token & 0xe0) >> 5)); int nLiterals = (int)((unsigned int)((token & 0x70) >> 4));
if (nLiterals < LITERALS_RUN_LEN) { if (nLiterals < LITERALS_RUN_LEN) {
memcpy(pCurOutData, pInBlock, 8); memcpy(pCurOutData, pInBlock, 8);
@ -184,20 +184,17 @@ int lzsa_expand_block(const unsigned char *pInBlock, int nBlockSize, unsigned ch
int nMatchOffset; int nMatchOffset;
nMatchOffset = ((unsigned int)*pInBlock++); nMatchOffset = ((unsigned int)*pInBlock++);
if (token & 0x01) { if (token & 0x80) {
if (pInBlock >= pInBlockEnd) return -1; if (pInBlock >= pInBlockEnd) return -1;
nMatchOffset |= (((unsigned int)*pInBlock++) << 8); nMatchOffset |= (((unsigned int)*pInBlock++) << 8);
if (nMatchOffset == 0) break;
}
else {
nMatchOffset++;
} }
nMatchOffset++;
const unsigned char *pSrc = pCurOutData - nMatchOffset; const unsigned char *pSrc = pCurOutData - nMatchOffset;
if (pSrc < pOutData) if (pSrc < pOutData)
return -1; return -1;
int nMatchLen = (int)((unsigned int)((token & 0x1e) >> 1)); int nMatchLen = (int)((unsigned int)(token & 0x0f));
if (nMatchLen < (16 - MIN_MATCH_SIZE + 1) && (pSrc + MIN_MATCH_SIZE + nMatchLen) < pCurOutData && pCurOutData < pOutDataFastEnd) { if (nMatchLen < (16 - MIN_MATCH_SIZE + 1) && (pSrc + MIN_MATCH_SIZE + nMatchLen) < pCurOutData && pCurOutData < pOutDataFastEnd) {
memcpy(pCurOutData, pSrc, 16); memcpy(pCurOutData, pSrc, 16);
pCurOutData += (MIN_MATCH_SIZE + nMatchLen); pCurOutData += (MIN_MATCH_SIZE + nMatchLen);
@ -213,7 +210,7 @@ int lzsa_expand_block(const unsigned char *pInBlock, int nBlockSize, unsigned ch
while (pInBlock < pInBlockEnd) { while (pInBlock < pInBlockEnd) {
const unsigned char token = *pInBlock++; const unsigned char token = *pInBlock++;
int nLiterals = (int)((unsigned int)((token & 0xe0) >> 5)); int nLiterals = (int)((unsigned int)((token & 0x70) >> 4));
if (lzsa_expand_literals_slow(&pInBlock, pInBlockEnd, nLiterals, &pCurOutData, pOutDataEnd)) if (lzsa_expand_literals_slow(&pInBlock, pInBlockEnd, nLiterals, &pCurOutData, pOutDataEnd))
return -1; return -1;
@ -222,20 +219,17 @@ int lzsa_expand_block(const unsigned char *pInBlock, int nBlockSize, unsigned ch
int nMatchOffset; int nMatchOffset;
nMatchOffset = ((unsigned int)*pInBlock++); nMatchOffset = ((unsigned int)*pInBlock++);
if (token & 0x01) { if (token & 0x80) {
if (pInBlock >= pInBlockEnd) return -1; if (pInBlock >= pInBlockEnd) return -1;
nMatchOffset |= (((unsigned int)*pInBlock++) << 8); nMatchOffset |= (((unsigned int)*pInBlock++) << 8);
if (nMatchOffset == 0) break;
}
else {
nMatchOffset++;
} }
nMatchOffset++;
const unsigned char *pSrc = pCurOutData - nMatchOffset; const unsigned char *pSrc = pCurOutData - nMatchOffset;
if (pSrc < pOutData) if (pSrc < pOutData)
return -1; return -1;
int nMatchLen = (int)((unsigned int)((token & 0x1e) >> 1)); int nMatchLen = (int)((unsigned int)(token & 0x0f));
if (lzsa_expand_match_slow(&pInBlock, pInBlockEnd, pSrc, nMatchLen, &pCurOutData, pOutDataEnd, pOutDataFastEnd)) if (lzsa_expand_match_slow(&pInBlock, pInBlockEnd, pSrc, nMatchLen, &pCurOutData, pOutDataEnd, pOutDataFastEnd))
return -1; return -1;
} }

42
src/main.c
View File

@ -223,16 +223,26 @@ static int lzsa_compress(const char *pszInFilename, const char *pszOutFilename,
} }
} }
unsigned char cFooter[3]; unsigned char cFooter[4];
int nFooterSize = ((nOptions & OPT_RAW) == 0) ? 3 : 2; int nFooterSize;
cFooter[0] = 0x00; /* EOD frame (written even in raw mode, so that the end of the data can be detected) */ if ((nOptions & OPT_RAW) != 0) {
cFooter[1] = 0x00; cFooter[0] = 0x00; /* EOD marker for raw block */
cFooter[2] = 0x00; cFooter[1] = 0xff;
cFooter[2] = 0x00;
cFooter[3] = 0x00;
nFooterSize = 4;
}
else {
cFooter[0] = 0x00; /* EOD frame */
cFooter[1] = 0x00;
cFooter[2] = 0x00;
nFooterSize = 3;
}
if (!bError) if (!bError)
bError = fwrite(cFooter, 1, nFooterSize, f_out) != nFooterSize; bError = fwrite(cFooter, 1, nFooterSize, f_out) != nFooterSize;
nCompressedSize += (long long) nFooterSize; nCompressedSize += (long long)nFooterSize;
if (!bError && (nOptions & OPT_VERBOSE)) { if (!bError && (nOptions & OPT_VERBOSE)) {
nEndTime = lzsa_get_time(); nEndTime = lzsa_get_time();
@ -305,7 +315,7 @@ static int lzsa_decompress(const char *pszInFilename, const char *pszOutFilename
nFileSize = (unsigned int)ftell(pInFile); nFileSize = (unsigned int)ftell(pInFile);
fseek(pInFile, 0, SEEK_SET); fseek(pInFile, 0, SEEK_SET);
if (nFileSize < 2) { if (nFileSize < 4) {
fclose(pInFile); fclose(pInFile);
pInFile = NULL; pInFile = NULL;
fprintf(stderr, "invalid file size for raw block mode\n"); fprintf(stderr, "invalid file size for raw block mode\n");
@ -371,15 +381,15 @@ static int lzsa_decompress(const char *pszInFilename, const char *pszOutFilename
(((unsigned int)cBlockSize[2]) << 16); (((unsigned int)cBlockSize[2]) << 16);
} }
else { else {
nBlockSize = 0; nBlockSize = 0xffffff;
} }
} }
else { else {
nBlockSize = nFileSize - 2; nBlockSize = nFileSize - 4;
nFileSize = 0; nFileSize = 0xffffff;
} }
if (nBlockSize != 0) { if ((nBlockSize & 0x400000) == 0) {
bool bIsUncompressed = (nBlockSize & 0x800000) != 0; bool bIsUncompressed = (nBlockSize & 0x800000) != 0;
int nDecompressedSize = 0; int nDecompressedSize = 0;
@ -483,7 +493,7 @@ static int lzsa_compare(const char *pszInFilename, const char *pszOutFilename, c
nFileSize = (unsigned int)ftell(pInFile); nFileSize = (unsigned int)ftell(pInFile);
fseek(pInFile, 0, SEEK_SET); fseek(pInFile, 0, SEEK_SET);
if (nFileSize < 2) { if (nFileSize < 4) {
fclose(pInFile); fclose(pInFile);
pInFile = NULL; pInFile = NULL;
fprintf(stderr, "invalid file size for raw block mode\n"); fprintf(stderr, "invalid file size for raw block mode\n");
@ -571,15 +581,15 @@ static int lzsa_compare(const char *pszInFilename, const char *pszOutFilename, c
(((unsigned int)cBlockSize[2]) << 16); (((unsigned int)cBlockSize[2]) << 16);
} }
else { else {
nBlockSize = 0; nBlockSize = 0xffffff;
} }
} }
else { else {
nBlockSize = nFileSize - 2; nBlockSize = nFileSize - 4;
nFileSize = 0; nFileSize = 0xffffff;
} }
if (nBlockSize != 0) { if ((nBlockSize & 0x400000) == 0) {
bool bIsUncompressed = (nBlockSize & 0x800000) != 0; bool bIsUncompressed = (nBlockSize & 0x800000) != 0;
int nDecompressedSize = 0; int nDecompressedSize = 0;

15
src/shrink.c
View File

@ -578,7 +578,7 @@ static void lzsa_optimize_command_count(lsza_compressor *pCompressor, const int
int nMatchOffset = pMatch->offset; int nMatchOffset = pMatch->offset;
int nMatchLen = pMatch->length; int nMatchLen = pMatch->length;
int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE; int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE;
int nNibbleLongOffset = (nMatchOffset <= 256) ? 0x00 : 0x01; int nNibbleLongOffset = (nMatchOffset <= 256) ? 0x00 : 0x80;
int nTokenSize = 1 /* nibble */ + lzsa_get_literals_varlen_size(nNumLiterals) + (nNibbleLongOffset ? 2 : 1) /* match offset */ + lzsa_get_match_varlen_size(nEncodedMatchLen); int nTokenSize = 1 /* nibble */ + lzsa_get_literals_varlen_size(nNumLiterals) + (nNibbleLongOffset ? 2 : 1) /* match offset */ + lzsa_get_match_varlen_size(nEncodedMatchLen);
if ((((nNumLiterals + nMatchLen) < LITERALS_RUN_LEN && nTokenSize >= nMatchLen) || if ((((nNumLiterals + nMatchLen) < LITERALS_RUN_LEN && nTokenSize >= nMatchLen) ||
@ -631,7 +631,7 @@ static int lzsa_write_block(lsza_compressor *pCompressor, const unsigned char *p
int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE; int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE;
int nNibbleLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN) ? LITERALS_RUN_LEN : nNumLiterals; int nNibbleLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN) ? LITERALS_RUN_LEN : nNumLiterals;
int nNibbleMatchLen = (nEncodedMatchLen >= MATCH_RUN_LEN) ? MATCH_RUN_LEN : nEncodedMatchLen; int nNibbleMatchLen = (nEncodedMatchLen >= MATCH_RUN_LEN) ? MATCH_RUN_LEN : nEncodedMatchLen;
int nNibbleLongOffset = (nMatchOffset <= 256) ? 0x00 : 0x01; int nNibbleLongOffset = (nMatchOffset <= 256) ? 0x00 : 0x80;
int nTokenSize = 1 /* nibble */ + lzsa_get_literals_varlen_size(nNumLiterals) + nNumLiterals + (nNibbleLongOffset ? 2 : 1) /* match offset */ + lzsa_get_match_varlen_size(nEncodedMatchLen); int nTokenSize = 1 /* nibble */ + lzsa_get_literals_varlen_size(nNumLiterals) + nNumLiterals + (nNibbleLongOffset ? 2 : 1) /* match offset */ + lzsa_get_match_varlen_size(nEncodedMatchLen);
if ((nOutOffset + nTokenSize) > nMaxOutDataSize) if ((nOutOffset + nTokenSize) > nMaxOutDataSize)
@ -639,7 +639,7 @@ static int lzsa_write_block(lsza_compressor *pCompressor, const unsigned char *p
if (nMatchOffset < MIN_OFFSET || nMatchOffset > MAX_OFFSET) if (nMatchOffset < MIN_OFFSET || nMatchOffset > MAX_OFFSET)
return -1; return -1;
pOutData[nOutOffset++] = (nNibbleLiteralsLen << 5) | (nNibbleMatchLen << 1) | nNibbleLongOffset; pOutData[nOutOffset++] = nNibbleLongOffset | (nNibbleLiteralsLen << 4) | nNibbleMatchLen;
nOutOffset = lzsa_write_literals_varlen(pOutData, nOutOffset, nNumLiterals); nOutOffset = lzsa_write_literals_varlen(pOutData, nOutOffset, nNumLiterals);
if (nNumLiterals != 0) { if (nNumLiterals != 0) {
@ -648,12 +648,9 @@ static int lzsa_write_block(lsza_compressor *pCompressor, const unsigned char *p
nNumLiterals = 0; nNumLiterals = 0;
} }
pOutData[nOutOffset++] = (nMatchOffset - 1) & 0xff;
if (nNibbleLongOffset) { if (nNibbleLongOffset) {
pOutData[nOutOffset++] = nMatchOffset & 0xff; pOutData[nOutOffset++] = (nMatchOffset - 1) >> 8;
pOutData[nOutOffset++] = nMatchOffset >> 8;
}
else {
pOutData[nOutOffset++] = (nMatchOffset - 1) & 0xff;
} }
nOutOffset = lzsa_write_match_varlen(pOutData, nOutOffset, nEncodedMatchLen); nOutOffset = lzsa_write_match_varlen(pOutData, nOutOffset, nEncodedMatchLen);
i += nMatchLen; i += nMatchLen;
@ -675,7 +672,7 @@ static int lzsa_write_block(lsza_compressor *pCompressor, const unsigned char *p
if ((nOutOffset + nTokenSize) > nMaxOutDataSize) if ((nOutOffset + nTokenSize) > nMaxOutDataSize)
return -1; return -1;
pOutData[nOutOffset++] = (nNibbleLiteralsLen << 5) | 0x01; pOutData[nOutOffset++] = (nNibbleLiteralsLen << 4) | 0x0f;
nOutOffset = lzsa_write_literals_varlen(pOutData, nOutOffset, nNumLiterals); nOutOffset = lzsa_write_literals_varlen(pOutData, nOutOffset, nNumLiterals);
if (nNumLiterals != 0) { if (nNumLiterals != 0) {