From ec32cb884acd5935ccf6d2760977b740765b00a5 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Thu, 9 Jul 2020 17:35:34 -0400 Subject: [PATCH 01/21] import: source code for the tool, WIP --- source/bctypes.h | 48 +++++++++++ source/c2_file.cpp | 143 ++++++++++++++++++++++++++++++ source/c2_file.h | 73 ++++++++++++++++ source/lzb.cpp | 210 +++++++++++++++++++++++++++++++++++++++++++++ source/lzb.h | 14 +++ source/main.cpp | 148 ++++++++++++++++++++++++++++++++ 6 files changed, 636 insertions(+) create mode 100644 source/bctypes.h create mode 100644 source/c2_file.cpp create mode 100644 source/c2_file.h create mode 100644 source/lzb.cpp create mode 100644 source/lzb.h create mode 100644 source/main.cpp diff --git a/source/bctypes.h b/source/bctypes.h new file mode 100644 index 0000000..7a55f9d --- /dev/null +++ b/source/bctypes.h @@ -0,0 +1,48 @@ +/* + bctypes.h + + Because I need Standard Types +*/ + +#ifndef _bctypes_h +#define _bctypes_h + +typedef signed char i8; +typedef unsigned char u8; +typedef signed short i16; +typedef unsigned short u16; +typedef signed long i32; +typedef unsigned long u32; + +typedef signed long long i64; +typedef unsigned long long u64; + +// If we're using C, I still like having a bool around +#ifndef __cplusplus +typedef i32 bool; +#define false (0) +#define true (!false) +#define nullptr 0 +#endif + +typedef float f32; +typedef float r32; +typedef double f64; +typedef double r64; + + +#define null (0) + +// Odd Types +typedef union { +// u128 ul128; + u64 ul64[2]; + u32 ui32[4]; +} QWdata; + + +#endif // _bctypes_h + +// EOF - bctypes.h + + diff --git a/source/c2_file.cpp b/source/c2_file.cpp new file mode 100644 index 0000000..6dab4b6 --- /dev/null +++ b/source/c2_file.cpp @@ -0,0 +1,143 @@ +// +// C++ Encoder/Decoder +// For C2, Paintworks Animation File Format +// +// File Format summary here from Brutal Deluxe +// +//0000.7fff first pic +//8000.8003 length of frame data - 8008 +//8004.8007 timing +//8008.800b length of first frame data +//800c.800d first frame index +//800e.800f first frame data +// +// Offset 0, Value FFFF indicates the end of a frame +// 00 00 FF FF +// +#include "c2_file.h" +#include + +// If these structs are the wrong size, there's an issue with type sizes, and +// your compiler +static_assert(sizeof(C2File_Header)==0x800C, "C2File_Header is supposed to be 0x800C bytes"); + +//------------------------------------------------------------------------------ +// Load in a FanFile constructor +// +C2File::C2File(const char *pFilePath) + : m_widthPixels(320) + , m_heightPixels(200) +{ + LoadFromFile(pFilePath); +} +//------------------------------------------------------------------------------ + +C2File::~C2File() +{ + // Free Up the memory + for (int idx = 0; idx < m_pC1PixelMaps.size(); ++idx) + { + delete[] m_pC1PixelMaps[idx]; + m_pC1PixelMaps[ idx ] = nullptr; + } +} + +//------------------------------------------------------------------------------ + +void C2File::LoadFromFile(const char* pFilePath) +{ + // Free Up the memory + for (int idx = 0; idx < m_pC1PixelMaps.size(); ++idx) + { + delete[] m_pC1PixelMaps[idx]; + m_pC1PixelMaps[ idx ] = nullptr; + } + + m_pC1PixelMaps.clear(); + //-------------------------------------------------------------------------- + + std::vector bytes; + + //-------------------------------------------------------------------------- + // Read the file into memory + FILE* pFile = nullptr; + errno_t err = fopen_s(&pFile, pFilePath, "rb"); + + if (0==err) + { + fseek(pFile, 0, SEEK_END); + size_t length = ftell(pFile); // get file size + fseek(pFile, 0, SEEK_SET); + + bytes.resize( length ); // make sure buffer is large enough + + // Read in the file + fread(&bytes[0], sizeof(unsigned char), bytes.size(), pFile); + fclose(pFile); + } + + if (bytes.size()) + { + size_t file_offset = 0; // File Cursor + + // Bytes are in the buffer, so let's start looking at what we have + C2File_Header* pHeader = (C2File_Header*) &bytes[0]; + + // Early out if things don't look right + if (!pHeader->IsValid((unsigned int)bytes.size())) + return; + + // Grab Initial Frame, and put it in the list + + unsigned char* pFrame = new unsigned char[ 0x8000 ]; + unsigned char* pCanvas = new unsigned char[ 0x8001 ]; // each frame changes the canvas + m_pC1PixelMaps.push_back(pFrame); + memcpy(pFrame, &bytes[0], 0x8000); + memcpy(pCanvas, &bytes[0], 0x8000); + + //---------------------------------------------------------------------- + // Process Frames as we encounter them + file_offset += sizeof(C2File_Header); + + // Since we always pull 4 bytes + // let's keep us from pulling bytes outside our buffer + size_t eof_size = bytes.size() - 4; + + // While we're not at the end of the file + unsigned short offset = 0; + //unsigned short prev_offset = 0; + unsigned short data = 0xFFFF; + while (file_offset <= eof_size) + { + //prev_offset = offset; + + offset = (unsigned short)bytes[ file_offset++ ]; + offset |= ((unsigned short)bytes[ file_offset++ ])<<8; + + data = (unsigned short)bytes[ file_offset++ ]; + data |= ((unsigned short)bytes[ file_offset++ ])<<8; + + //if (((offset == 0)&&(data == 0xFFFF))||(offset < prev_offset)) + if (0 == offset) + { + // End of Frame, capture a copy + pFrame = new unsigned char[ 0x8000 ]; + memcpy(pFrame, pCanvas, 0x8000); + m_pC1PixelMaps.push_back(pFrame); + } + else + { + // Apply Change to the Canvas + offset &= 0x7FFF; // force the offset into the Canvas + // probably should ignore offsets outside the canvas + pCanvas[ offset ] = data & 0xFF; + pCanvas[ offset+1 ] = (data >> 8) & 0xFF; + } + } + + delete[] pCanvas; + } +} + +//------------------------------------------------------------------------------ + diff --git a/source/c2_file.h b/source/c2_file.h new file mode 100644 index 0000000..954321f --- /dev/null +++ b/source/c2_file.h @@ -0,0 +1,73 @@ +// +// C++ Encoder/Decoder +// For C2, Paintworks Animation File Format +// +// File Format summary here from Brutal Deluxe +// +//0000.7fff first pic +//8000.8003 length of frame data - 8008 +//8004.8007 timing +//8008.800b length of first frame data +//800c.800d first frame index +//800e.800f first frame data +// +// Offset 0, Value FFFF indicates the end of a frame +// 00 00 FF FF +// +#ifndef C2_FILE_H +#define C2_FILE_H + +#include + +#pragma pack(push, 1) + +typedef struct C2File_Header +{ + char image[0x8000]; // C1 Initial Image + + unsigned int file_length; // length of frame_data (file length - 8008) + unsigned int timing; + unsigned int frame_length; // first encoded frame length + +//------------------------------------------------------------------------------ +// If you're doing C, just get rid of these methods + bool IsValid(unsigned int fileLength) + { + if ((file_length+0x8008) != fileLength) + return false; // size isn't right + + return true; + } + +} C2File_Header; + +#pragma pack(pop) + +class C2File +{ +public: + // Load in a C2 File + C2File(const char *pFilePath); + + ~C2File(); + + // Retrieval + void LoadFromFile(const char* pFilePath); + int GetFrameCount() { return (int)m_pC1PixelMaps.size(); } + int GetWidth() { return m_widthPixels; } + int GetHeight() { return m_heightPixels; } + + const std::vector& GetPixelMaps() { return m_pC1PixelMaps; } + +private: + + int m_widthPixels; // Width of image in pixels + int m_heightPixels; // Height of image in pixels + + std::vector m_pC1PixelMaps; + +}; + + +#endif // C2_FILE_H + diff --git a/source/lzb.cpp b/source/lzb.cpp new file mode 100644 index 0000000..57a26b6 --- /dev/null +++ b/source/lzb.cpp @@ -0,0 +1,210 @@ +// +// LZB Encode / Decode +// +#include "lzb.h" + +#include +#include + +#include "bctypes.h" + +#define DICTIONARY_SIZE (32 * 1024) +// +// Yes This is a 32K Buffer, of bytes, with no structure to it +// +static unsigned char Dictionary[ DICTIONARY_SIZE ]; + +static int AddDictionary(const std::vector&data, int dictionarySize); +static int EmitLiteral(unsigned char *pDest, std::vector& data); +static int EmitReference(unsigned char *pDest, int dictionaryOffset, std::vector& data); +static int DictionaryMatch(const std::vector& data, int dictionarySize); + +int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) +{ + printf("LZB_Compress %d bytes\n", sourceSize); + + // anything less than 3 bytes, is going to be a literal match + + int processedBytes = 0; + int bytesInDictionary = 0; + int bytesEmitted = 0; + + std::vector candidate_data; + + while (processedBytes < sourceSize) + { + unsigned char byte_data = pSource[ processedBytes++ ]; + candidate_data.push_back(byte_data); + + // The dictionary only contains bytes that have been emitted, so we + // can't add this byte until we've emitted it? + + if (candidate_data.size() < 3) continue; + + if (DictionaryMatch(candidate_data, bytesInDictionary) < 0) + { + // Was there a dictionary match + std::vector prev_data = candidate_data; + prev_data.pop_back(); + + int MatchOffset = DictionaryMatch(prev_data, bytesInDictionary); + + if ((MatchOffset >= 0) && prev_data.size() > 2) + { + bytesInDictionary = AddDictionary(prev_data, bytesInDictionary); + bytesEmitted += EmitReference(pDest + bytesEmitted, MatchOffset, prev_data); + candidate_data[0] = candidate_data[ candidate_data.size() - 1 ]; + candidate_data.resize(1); + } + else + { + // Add Dictionary + bytesInDictionary = AddDictionary(candidate_data, bytesInDictionary); + bytesEmitted += EmitLiteral(pDest + bytesEmitted, candidate_data); + } + } + } + + if (candidate_data.size() > 0) + { + // Emit as a literal? (we have 1 more chance here for a match + // Add Dictionary + bytesInDictionary = AddDictionary(candidate_data, bytesInDictionary); + bytesEmitted += EmitLiteral(pDest + bytesEmitted, candidate_data); + } + + return bytesEmitted; +} + +//------------------------------------------------------------------------------ +// Return new dictionarySize +static int AddDictionary(const std::vector&data, int dictionarySize) +{ + int dataIndex = 0; + + while ((dictionarySize < DICTIONARY_SIZE) && (dataIndex < data.size())) + { + Dictionary[ dictionarySize++ ] = data[ dataIndex++ ]; + } + + return dictionarySize; +} + +//------------------------------------------------------------------------------ +// +// Return offset into dictionary where the string matches +// +// -1 means, no match +// +static int DictionaryMatch(const std::vector& data, int dictionarySize) +{ + if(dictionarySize < data.size()) + { + return -1; + } + + int dictionaryOffset = 0; + + int result = -1; + + // Check the dictionary for a match, brute force + for (int idx = 0; idx <= (dictionarySize-data.size()); ++idx) + { + bool bMatch = true; + for (int dataIdx = 0; dataIdx < data.size(); ++dataIdx) + { + if (data[ dataIdx ] == Dictionary[ idx + dataIdx ]) + continue; + + bMatch = false; + break; + } + + if (bMatch) + { + result = idx; + break; + } + } + + return result; +} + +//------------------------------------------------------------------------------ + +static int EmitLiteral(unsigned char *pDest, std::vector& data) +{ + // Return Size + int outSize = 2 + (int)data.size(); + + // Opcode + *pDest++ = (unsigned char)(data.size() & 0xFF); + *pDest++ = (unsigned char)((data.size() >> 8) & 0x7F); + + // Literal Data + for (int idx = 0; idx < data.size(); ++idx) + { + *pDest++ = data[ idx ]; + } + + data.clear(); + + return outSize; +} + +//------------------------------------------------------------------------------ + +static int EmitReference(unsigned char *pDest, int dictionaryOffset, std::vector& data) +{ + // Return Size + int outSize = 2 + 2; + + // Opcode + *pDest++ = (unsigned char)(data.size() & 0xFF); + *pDest++ = (unsigned char)((data.size() >> 8) & 0x7F) | 0x80; + + *pDest++ = (unsigned char)(dictionaryOffset & 0xFF); + *pDest++ = (unsigned char)((dictionaryOffset>>8) & 0xFF); + + data.clear(); + + return outSize; +} + +//------------------------------------------------------------------------------ +// +// Simple Decompress, for validation +// +void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) +{ + int decompressedBytes = 0; + + while (decompressedBytes < destSize) + { + u16 opcode = *pSource++; + opcode |= ((u16)(*pSource++))<<8; + + if (opcode & 0x8000) + { + // Dictionary + opcode &= 0x7FFF; + + // Dictionary Copy from the output stream + u16 offset = *pSource++; + offset |= ((u16)(*pSource++))<<8; + + memcpy(&pDest[ decompressedBytes ], &pDest[ offset ], opcode); + decompressedBytes += opcode; + } + else + { + // Literal Copy, from compressed stream + memcpy(&pDest[ decompressedBytes ], pSource, opcode); + decompressedBytes += opcode; + pSource += opcode; + } + } +} + +//------------------------------------------------------------------------------ + diff --git a/source/lzb.h b/source/lzb.h new file mode 100644 index 0000000..8284aa2 --- /dev/null +++ b/source/lzb.h @@ -0,0 +1,14 @@ +// +// LZB Encode / Decode +// +#ifndef LZB_H +#define LZB_H + +// +// returns the size of data saved into the pDest Buffer +// +int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize); +void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize); + +#endif // LZB_H + diff --git a/source/main.cpp b/source/main.cpp new file mode 100644 index 0000000..fcb0d00 --- /dev/null +++ b/source/main.cpp @@ -0,0 +1,148 @@ +// +// GSLA - GS LZB Animation Tool +// + +#include +#include +#include + +#include "c2_file.h" +#include "lzb.h" + +//------------------------------------------------------------------------------ +static void helpText() +{ + printf("GSLA - v0.0\n"); + printf("--------------\n"); + printf("GS Lzb Animation Tool\n"); + printf("\n"); + printf("\ngsla [options] \n"); + + exit(-1); +} +//------------------------------------------------------------------------------ + +//------------------------------------------------------------------------------ +// Local helper functions + +static std::string toLower(const std::string s) +{ + std::string result = s; + + for (int idx = 0; idx < result.size(); ++idx) + { + result[ idx ] = (char)tolower(result[idx]); + } + + return result; +} + +// Case Insensitive +static bool endsWith(const std::string& S, const std::string& SUFFIX) +{ + bool bResult = false; + + std::string s = toLower(S); + std::string suffix = toLower(SUFFIX); + + bResult = s.rfind(suffix) == (s.size()-suffix.size()); + + return bResult; +} +//------------------------------------------------------------------------------ + + +int main(int argc, char* argv[]) +{ + char* pInfilePath = nullptr; + char* pOutfilePath = nullptr; + + + // index 0 is the executable name + + if (argc < 2) helpText(); + + for (int idx = 1; idx < argc; ++idx ) + { + char* arg = argv[ idx ]; + + if ('-' == arg[0]) + { + // Parse as an option + // Currently I have no options, so I'll just skip + } + else if (nullptr == pInfilePath) + { + // Assume the first non-option is an input file path + pInfilePath = argv[ idx ]; + } + else if (nullptr == pOutfilePath) + { + // Assume second non-option is an output file path + pOutfilePath = argv[ idx ]; + } + else + { + // Oh Crap, we have a non-option, but we don't know what to do with + // it + printf("ERROR: Invalid option, Arg %d = %s\n\n", idx, argv[ idx ]); + helpText(); + } + } + + if (pInfilePath) + { + // See what we can do with the input file path + // could be a .gsla file, for a .c2 file, or maybe a series of .c1 files + if (endsWith(pInfilePath, ".c2") || endsWith(pInfilePath, "#c20000")) + { + // It's a C2 file + + printf("Loading C2 File %s\n", pInfilePath); + + C2File c2data( pInfilePath ); + + int frameCount = c2data.GetFrameCount(); + + if (frameCount < 1) + { + // c2 file can't be valid, if there are no frames + printf("C2 File seems invalid.\n"); + helpText(); + } + + const std::vector& c1Datas = c2data.GetPixelMaps(); + + unsigned char workbuffer[64*1024]; + + for (int idx = 0; idx < frameCount; ++idx) + { + int compressedSize = LZB_Compress(workbuffer, c1Datas[ idx ], 32 * 1024); + printf("compressedSize = %d\n", compressedSize); + + unsigned char validationBuffer[ 32 * 1024 ]; + + LZB_Decompress(validationBuffer, workbuffer, 32 * 1024); + + if (0 == memcmp(c1Datas[ idx ], validationBuffer, 32*1024)) + { + printf("Decompression Validated\n"); + } + else + { + printf("Decompression Corrupted\n"); + } + + } + } + + } + else + { + helpText(); + } + + + return 0; +} + From e2676ed7bd9ae659f0fdace27c8b15ce6743a185 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Thu, 9 Jul 2020 17:37:39 -0400 Subject: [PATCH 02/21] import: visual studio project files --- .gitignore | 7 ++ vcxproj/gsla.sln | 31 +++++++ vcxproj/gsla.vcxproj | 154 +++++++++++++++++++++++++++++++++++ vcxproj/gsla.vcxproj.filters | 35 ++++++++ 4 files changed, 227 insertions(+) create mode 100644 .gitignore create mode 100644 vcxproj/gsla.sln create mode 100644 vcxproj/gsla.vcxproj create mode 100644 vcxproj/gsla.vcxproj.filters diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6b45cb4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.user +*.vpj +*.vpw +*.vpwhist +*.vtg +vcxproj/.vs/ +vcxproj/x64/ diff --git a/vcxproj/gsla.sln b/vcxproj/gsla.sln new file mode 100644 index 0000000..8656bea --- /dev/null +++ b/vcxproj/gsla.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30225.117 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "gsla", "gsla.vcxproj", "{08214015-2B09-4ED2-ACD4-93031A939CF1}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {08214015-2B09-4ED2-ACD4-93031A939CF1}.Debug|x64.ActiveCfg = Debug|x64 + {08214015-2B09-4ED2-ACD4-93031A939CF1}.Debug|x64.Build.0 = Debug|x64 + {08214015-2B09-4ED2-ACD4-93031A939CF1}.Debug|x86.ActiveCfg = Debug|Win32 + {08214015-2B09-4ED2-ACD4-93031A939CF1}.Debug|x86.Build.0 = Debug|Win32 + {08214015-2B09-4ED2-ACD4-93031A939CF1}.Release|x64.ActiveCfg = Release|x64 + {08214015-2B09-4ED2-ACD4-93031A939CF1}.Release|x64.Build.0 = Release|x64 + {08214015-2B09-4ED2-ACD4-93031A939CF1}.Release|x86.ActiveCfg = Release|Win32 + {08214015-2B09-4ED2-ACD4-93031A939CF1}.Release|x86.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {922EAE98-7805-4172-A57D-9B441077B035} + EndGlobalSection +EndGlobal diff --git a/vcxproj/gsla.vcxproj b/vcxproj/gsla.vcxproj new file mode 100644 index 0000000..3e0d09f --- /dev/null +++ b/vcxproj/gsla.vcxproj @@ -0,0 +1,154 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 16.0 + Win32Proj + {08214015-2b09-4ed2-acd4-93031a939cf1} + gsla + 10.0 + + + + Application + true + v142 + Unicode + + + Application + false + v142 + true + Unicode + + + Application + true + v142 + Unicode + + + Application + false + v142 + true + Unicode + + + + + + + + + + + + + + + + + + + + + true + + + false + + + true + + + false + + + + Level3 + true + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + + + + + Level3 + true + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + true + true + + + + + Level3 + true + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + + + + + Level3 + true + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + true + true + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/vcxproj/gsla.vcxproj.filters b/vcxproj/gsla.vcxproj.filters new file mode 100644 index 0000000..3ab00b7 --- /dev/null +++ b/vcxproj/gsla.vcxproj.filters @@ -0,0 +1,35 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;c++;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd + + + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file From 0c5c466f64a70b2013ec97d8dd7b4ace8c1feb3c Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Mon, 13 Jul 2020 16:07:12 -0400 Subject: [PATCH 03/21] LZB Encoder: produces better compression (actually generates run/length pattern encoding) --- source/lzb.cpp | 139 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 129 insertions(+), 10 deletions(-) diff --git a/source/lzb.cpp b/source/lzb.cpp index 57a26b6..325d2e2 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -16,6 +16,7 @@ static unsigned char Dictionary[ DICTIONARY_SIZE ]; static int AddDictionary(const std::vector&data, int dictionarySize); static int EmitLiteral(unsigned char *pDest, std::vector& data); +static int ConcatLiteral(unsigned char *pDest, std::vector& data); static int EmitReference(unsigned char *pDest, int dictionaryOffset, std::vector& data); static int DictionaryMatch(const std::vector& data, int dictionarySize); @@ -29,6 +30,10 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) int bytesInDictionary = 0; int bytesEmitted = 0; + // dumb last emit is a literal stuff + bool bLastEmitIsLiteral = false; + int lastEmittedLiteralOffset = 0; + std::vector candidate_data; while (processedBytes < sourceSize) @@ -39,8 +44,6 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) // The dictionary only contains bytes that have been emitted, so we // can't add this byte until we've emitted it? - if (candidate_data.size() < 3) continue; - if (DictionaryMatch(candidate_data, bytesInDictionary) < 0) { // Was there a dictionary match @@ -49,28 +52,61 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) int MatchOffset = DictionaryMatch(prev_data, bytesInDictionary); - if ((MatchOffset >= 0) && prev_data.size() > 2) + if ((MatchOffset >= 0) && prev_data.size() > 3) { bytesInDictionary = AddDictionary(prev_data, bytesInDictionary); bytesEmitted += EmitReference(pDest + bytesEmitted, MatchOffset, prev_data); candidate_data[0] = candidate_data[ candidate_data.size() - 1 ]; candidate_data.resize(1); + bLastEmitIsLiteral = false; } else { // Add Dictionary bytesInDictionary = AddDictionary(candidate_data, bytesInDictionary); - bytesEmitted += EmitLiteral(pDest + bytesEmitted, candidate_data); + + if (bLastEmitIsLiteral) + { + // If the last emit was a literal, I want to concatenate + // this literal into the previous opcode, to save space + bytesEmitted += ConcatLiteral(pDest + lastEmittedLiteralOffset, candidate_data); + } + else + { + lastEmittedLiteralOffset = bytesEmitted; + bytesEmitted += EmitLiteral(pDest + bytesEmitted, candidate_data); + } + bLastEmitIsLiteral = true; } } } if (candidate_data.size() > 0) { - // Emit as a literal? (we have 1 more chance here for a match - // Add Dictionary - bytesInDictionary = AddDictionary(candidate_data, bytesInDictionary); - bytesEmitted += EmitLiteral(pDest + bytesEmitted, candidate_data); + + int MatchOffset = DictionaryMatch(candidate_data, bytesInDictionary); + + if ((MatchOffset >=0) && candidate_data.size() > 2) + { + bytesInDictionary = AddDictionary(candidate_data, bytesInDictionary); + bytesEmitted += EmitReference(pDest + bytesEmitted, MatchOffset, candidate_data); + } + else + { + // Add Dictionary + bytesInDictionary = AddDictionary(candidate_data, bytesInDictionary); + + if (bLastEmitIsLiteral) + { + // If the last emit was a literal, I want to concatenate + // this literal into the previous opcode, to save space + bytesEmitted += ConcatLiteral(pDest + lastEmittedLiteralOffset, candidate_data); + } + else + { + bytesEmitted += EmitLiteral(pDest + bytesEmitted, candidate_data); + } + } } return bytesEmitted; @@ -98,7 +134,45 @@ static int AddDictionary(const std::vector&data, int dictionarySi // static int DictionaryMatch(const std::vector& data, int dictionarySize) { - if(dictionarySize < data.size()) + if( (0 == dictionarySize ) || + (0 == data.size()) || + (data.size() > 16384) ) // 16384 is largest string copy we can encode + { + return -1; + } + + // Check the end of the dictionary, to see if this data could be a + // pattern "run" (where we can repeat a pattern for X many times for free + // using the memcpy with overlapping source/dest buffers) + // (This is a dictionary based pattern run/length) + + { + // Check for pattern sizes, start small + int max_pattern_size = dictionarySize; + + for (int pattern_size = 1; pattern_size <= max_pattern_size; ++pattern_size) + { + bool bMatch = true; + int pattern_start = dictionarySize - pattern_size; + + for (int dataIndex = 0; dataIndex < data.size(); ++dataIndex) + { + if (data[ dataIndex ] == Dictionary[ pattern_start + (dataIndex % pattern_size) ]) + continue; + + bMatch = false; + break; + } + + if (bMatch) + { + // Return a RLE Style match result + return pattern_start; + } + } + } + + if (dictionarySize < data.size()) { return -1; } @@ -130,6 +204,38 @@ static int DictionaryMatch(const std::vector& data, int dictionar return result; } +//------------------------------------------------------------------------------ +// +// Emit a literal, that appends itself to an existing literal +// +static int ConcatLiteral(unsigned char *pDest, std::vector& data) +{ + // Return Size + int outSize = (int)data.size(); + + int opCode = pDest[0]; + opCode |= (int)(((pDest[1])&0x7F)<<8); + + int skip = opCode; + opCode += outSize; + + // Opcode + *pDest++ = (unsigned char)(opCode & 0xFF); + *pDest++ = (unsigned char)((opCode >> 8) & 0x7F); + + pDest += skip; + + // Literal Data + for (int idx = 0; idx < data.size(); ++idx) + { + *pDest++ = data[ idx ]; + } + + data.clear(); + + return outSize; +} + //------------------------------------------------------------------------------ static int EmitLiteral(unsigned char *pDest, std::vector& data) @@ -171,6 +277,19 @@ static int EmitReference(unsigned char *pDest, int dictionaryOffset, std::vector return outSize; } +//------------------------------------------------------------------------------ +// +// Std C memcpy seems to be stopping this from happening +// probably for my protection +// +void mymemcpy(u8* pDest, u8* pSrc, int length) +{ + while (length-- > 0) + { + *pDest++ = *pSrc++; + } +} + //------------------------------------------------------------------------------ // // Simple Decompress, for validation @@ -193,7 +312,7 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) u16 offset = *pSource++; offset |= ((u16)(*pSource++))<<8; - memcpy(&pDest[ decompressedBytes ], &pDest[ offset ], opcode); + mymemcpy(&pDest[ decompressedBytes ], &pDest[ offset ], opcode); decompressedBytes += opcode; } else From e7198c6f4d540530a161da152896dc54da42c976 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Mon, 13 Jul 2020 16:41:19 -0400 Subject: [PATCH 04/21] LZB: add some notes for me --- source/lzb.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/lzb.cpp b/source/lzb.cpp index 325d2e2..67cf47d 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -41,9 +41,13 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) unsigned char byte_data = pSource[ processedBytes++ ]; candidate_data.push_back(byte_data); + // If there's a match, then add to the candidate data, and see if + // there's a bigger match (use previous result to speed up search) + // (KMP is probably the last planned optmization here) + + // The dictionary only contains bytes that have been emitted, so we // can't add this byte until we've emitted it? - if (DictionaryMatch(candidate_data, bytesInDictionary) < 0) { // Was there a dictionary match @@ -279,10 +283,10 @@ static int EmitReference(unsigned char *pDest, int dictionaryOffset, std::vector //------------------------------------------------------------------------------ // -// Std C memcpy seems to be stopping this from happening -// probably for my protection +// Std C memcpy seems to be stopping the copy from happening, when I overlap +// the buffer to get a pattern run copy (overlapped buffers) // -void mymemcpy(u8* pDest, u8* pSrc, int length) +static void my_memcpy(u8* pDest, u8* pSrc, int length) { while (length-- > 0) { @@ -312,7 +316,7 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) u16 offset = *pSource++; offset |= ((u16)(*pSource++))<<8; - mymemcpy(&pDest[ decompressedBytes ], &pDest[ offset ], opcode); + my_memcpy(&pDest[ decompressedBytes ], &pDest[ offset ], opcode); decompressedBytes += opcode; } else From 553c5775a96f6900465baf4703238e2b861fc58c Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Mon, 13 Jul 2020 16:51:45 -0400 Subject: [PATCH 05/21] more notes for me --- source/lzb.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/lzb.cpp b/source/lzb.cpp index 67cf47d..5ca0566 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -41,8 +41,17 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) unsigned char byte_data = pSource[ processedBytes++ ]; candidate_data.push_back(byte_data); + // Basic Flow Idea Here // If there's a match, then add to the candidate data, and see if // there's a bigger match (use previous result to speed up search) + // else + // if there's a previous match, and it's large enough, emit that + // else emit what we have as a literal + + + + + // (KMP is probably the last planned optmization here) From d28efbc390279d642382022d36e9fe8109f45b02 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Wed, 15 Jul 2020 13:14:27 -0400 Subject: [PATCH 06/21] remove use of vector template, and stop copying data around that doesn't need copied (substantially speeds up the code), even though we're still brute force --- source/lzb.cpp | 130 +++++++++++++++++++++++++++---------------------- 1 file changed, 73 insertions(+), 57 deletions(-) diff --git a/source/lzb.cpp b/source/lzb.cpp index 5ca0566..3138aef 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -4,42 +4,52 @@ #include "lzb.h" #include -#include +#include #include "bctypes.h" -#define DICTIONARY_SIZE (32 * 1024) +#define MAX_DICTIONARY_SIZE (32 * 1024) // // Yes This is a 32K Buffer, of bytes, with no structure to it // -static unsigned char Dictionary[ DICTIONARY_SIZE ]; +static unsigned char *pDictionary = nullptr; + +struct DataString { + int size; + unsigned char *pData; +}; + +static int AddDictionary(const DataString& data, int dictionarySize); +static int EmitLiteral(unsigned char *pDest, DataString& data); +static int ConcatLiteral(unsigned char *pDest, DataString& data); +static int EmitReference(unsigned char *pDest, int dictionaryOffset, DataString& data); +static int DictionaryMatch(const DataString& data, int dictionarySize); -static int AddDictionary(const std::vector&data, int dictionarySize); -static int EmitLiteral(unsigned char *pDest, std::vector& data); -static int ConcatLiteral(unsigned char *pDest, std::vector& data); -static int EmitReference(unsigned char *pDest, int dictionaryOffset, std::vector& data); -static int DictionaryMatch(const std::vector& data, int dictionarySize); int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) { printf("LZB_Compress %d bytes\n", sourceSize); - // anything less than 3 bytes, is going to be a literal match + // Initialize Dictionary + int bytesInDictionary = 0; // eventually add the ability to start with the dictionary filled + pDictionary = pSource; int processedBytes = 0; - int bytesInDictionary = 0; int bytesEmitted = 0; // dumb last emit is a literal stuff bool bLastEmitIsLiteral = false; int lastEmittedLiteralOffset = 0; - std::vector candidate_data; + DataString candidate_data; + candidate_data.pData = pSource; + candidate_data.size = 0; while (processedBytes < sourceSize) { - unsigned char byte_data = pSource[ processedBytes++ ]; - candidate_data.push_back(byte_data); + // Add a byte to the candidate_data, also tally number of processed + processedBytes++; + candidate_data.size++; // Basic Flow Idea Here // If there's a match, then add to the candidate data, and see if @@ -49,9 +59,6 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) // else emit what we have as a literal - - - // (KMP is probably the last planned optmization here) @@ -60,21 +67,23 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) if (DictionaryMatch(candidate_data, bytesInDictionary) < 0) { // Was there a dictionary match - std::vector prev_data = candidate_data; - prev_data.pop_back(); - int MatchOffset = DictionaryMatch(prev_data, bytesInDictionary); + // Previous Data, we can't get here with candidate_data.size == 0 + // this is an opportunity to use an assert + candidate_data.size--; - if ((MatchOffset >= 0) && prev_data.size() > 3) + int MatchOffset = DictionaryMatch(candidate_data, bytesInDictionary); + + if ((MatchOffset >= 0) && candidate_data.size > 3) { - bytesInDictionary = AddDictionary(prev_data, bytesInDictionary); - bytesEmitted += EmitReference(pDest + bytesEmitted, MatchOffset, prev_data); - candidate_data[0] = candidate_data[ candidate_data.size() - 1 ]; - candidate_data.resize(1); + processedBytes--; + bytesInDictionary = AddDictionary(candidate_data, bytesInDictionary); + bytesEmitted += EmitReference(pDest + bytesEmitted, MatchOffset, candidate_data); bLastEmitIsLiteral = false; } else { + candidate_data.size++; // Add Dictionary bytesInDictionary = AddDictionary(candidate_data, bytesInDictionary); @@ -94,12 +103,12 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) } } - if (candidate_data.size() > 0) + if (candidate_data.size > 0) { int MatchOffset = DictionaryMatch(candidate_data, bytesInDictionary); - if ((MatchOffset >=0) && candidate_data.size() > 2) + if ((MatchOffset >=0) && candidate_data.size > 2) { bytesInDictionary = AddDictionary(candidate_data, bytesInDictionary); bytesEmitted += EmitReference(pDest + bytesEmitted, MatchOffset, candidate_data); @@ -127,14 +136,15 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) //------------------------------------------------------------------------------ // Return new dictionarySize -static int AddDictionary(const std::vector&data, int dictionarySize) +static int AddDictionary(const DataString& data, int dictionarySize) { - int dataIndex = 0; + //int dataIndex = 0; + //while ((dictionarySize < MAX_DICTIONARY_SIZE) && (dataIndex < data.size)) + //{ + // pDictionary[ dictionarySize++ ] = data.pData[ dataIndex++ ]; + //} - while ((dictionarySize < DICTIONARY_SIZE) && (dataIndex < data.size())) - { - Dictionary[ dictionarySize++ ] = data[ dataIndex++ ]; - } + dictionarySize += data.size; return dictionarySize; } @@ -145,11 +155,11 @@ static int AddDictionary(const std::vector&data, int dictionarySi // // -1 means, no match // -static int DictionaryMatch(const std::vector& data, int dictionarySize) +static int DictionaryMatch(const DataString& data, int dictionarySize) { if( (0 == dictionarySize ) || - (0 == data.size()) || - (data.size() > 16384) ) // 16384 is largest string copy we can encode + (0 == data.size) || + (data.size > 16384) ) // 16384 is largest string copy we can encode { return -1; } @@ -168,9 +178,9 @@ static int DictionaryMatch(const std::vector& data, int dictionar bool bMatch = true; int pattern_start = dictionarySize - pattern_size; - for (int dataIndex = 0; dataIndex < data.size(); ++dataIndex) + for (int dataIndex = 0; dataIndex < data.size; ++dataIndex) { - if (data[ dataIndex ] == Dictionary[ pattern_start + (dataIndex % pattern_size) ]) + if (data.pData[ dataIndex ] == pDictionary[ pattern_start + (dataIndex % pattern_size) ]) continue; bMatch = false; @@ -185,7 +195,7 @@ static int DictionaryMatch(const std::vector& data, int dictionar } } - if (dictionarySize < data.size()) + if (dictionarySize < data.size) { return -1; } @@ -195,12 +205,12 @@ static int DictionaryMatch(const std::vector& data, int dictionar int result = -1; // Check the dictionary for a match, brute force - for (int idx = 0; idx <= (dictionarySize-data.size()); ++idx) + for (int idx = 0; idx <= (dictionarySize-data.size); ++idx) { bool bMatch = true; - for (int dataIdx = 0; dataIdx < data.size(); ++dataIdx) + for (int dataIdx = 0; dataIdx < data.size; ++dataIdx) { - if (data[ dataIdx ] == Dictionary[ idx + dataIdx ]) + if (data.pData[ dataIdx ] == pDictionary[ idx + dataIdx ]) continue; bMatch = false; @@ -221,10 +231,10 @@ static int DictionaryMatch(const std::vector& data, int dictionar // // Emit a literal, that appends itself to an existing literal // -static int ConcatLiteral(unsigned char *pDest, std::vector& data) +static int ConcatLiteral(unsigned char *pDest, DataString& data) { // Return Size - int outSize = (int)data.size(); + int outSize = (int)data.size; int opCode = pDest[0]; opCode |= (int)(((pDest[1])&0x7F)<<8); @@ -239,53 +249,59 @@ static int ConcatLiteral(unsigned char *pDest, std::vector& data) pDest += skip; // Literal Data - for (int idx = 0; idx < data.size(); ++idx) + for (int idx = 0; idx < data.size; ++idx) { - *pDest++ = data[ idx ]; + *pDest++ = data.pData[ idx ]; } - data.clear(); + // Clear + data.pData += data.size; + data.size = 0; return outSize; } //------------------------------------------------------------------------------ -static int EmitLiteral(unsigned char *pDest, std::vector& data) +static int EmitLiteral(unsigned char *pDest, DataString& data) { // Return Size - int outSize = 2 + (int)data.size(); + int outSize = 2 + (int)data.size; // Opcode - *pDest++ = (unsigned char)(data.size() & 0xFF); - *pDest++ = (unsigned char)((data.size() >> 8) & 0x7F); + *pDest++ = (unsigned char)(data.size & 0xFF); + *pDest++ = (unsigned char)((data.size >> 8) & 0x7F); // Literal Data - for (int idx = 0; idx < data.size(); ++idx) + for (int idx = 0; idx < data.size; ++idx) { - *pDest++ = data[ idx ]; + *pDest++ = data.pData[ idx ]; } - data.clear(); + // Clear + data.pData += data.size; + data.size = 0; return outSize; } //------------------------------------------------------------------------------ -static int EmitReference(unsigned char *pDest, int dictionaryOffset, std::vector& data) +static int EmitReference(unsigned char *pDest, int dictionaryOffset, DataString& data) { // Return Size int outSize = 2 + 2; // Opcode - *pDest++ = (unsigned char)(data.size() & 0xFF); - *pDest++ = (unsigned char)((data.size() >> 8) & 0x7F) | 0x80; + *pDest++ = (unsigned char)(data.size & 0xFF); + *pDest++ = (unsigned char)((data.size >> 8) & 0x7F) | 0x80; *pDest++ = (unsigned char)(dictionaryOffset & 0xFF); *pDest++ = (unsigned char)((dictionaryOffset>>8) & 0xFF); - data.clear(); + // Clear + data.pData += data.size; + data.size = 0; return outSize; } From e358c2a937513f95a888ba07954ded6fae7bebbd Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Wed, 15 Jul 2020 18:28:56 -0400 Subject: [PATCH 07/21] lzb: make it faster (WIP) --- source/lzb.cpp | 108 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 105 insertions(+), 3 deletions(-) diff --git a/source/lzb.cpp b/source/lzb.cpp index 3138aef..0e46214 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -15,6 +15,7 @@ static unsigned char *pDictionary = nullptr; struct DataString { + // Information about the data we're trying to match int size; unsigned char *pData; }; @@ -25,8 +26,79 @@ static int ConcatLiteral(unsigned char *pDest, DataString& data); static int EmitReference(unsigned char *pDest, int dictionaryOffset, DataString& data); static int DictionaryMatch(const DataString& data, int dictionarySize); +// Stuff I need for a faster version +static DataString LongestMatch(const DataString& data, const DataString* dictionary); +// +// New Version, still Brute Force, but not as many times +// int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) +{ + printf("LZB Compress %d bytes\n", sourceSize); + + unsigned char *pOriginalDest = pDest; + + DataString sourceData; + DataString dictionaryData; + DataString candidateData; + + // Source Data Stream - will compress until the size is zero + sourceData.pData = pSource; + sourceData.size = sourceSize; + + // Remember, this eventually will point at the frame buffer + dictionaryData.pData = pSource + dictionaryData.size = 0; + + // dumb last emit is a literal stuff + bool bLastEmitIsLiteral = false; + unsigned char* pLastLiteralDest = nullptr; + + while (sourceData.size > 0) + { + candidateData = LongestMatch(sourceData, dictionaryData); + + if (0 == candidateData.size) + { + candidateData.size = 1; + candidateData.pData = sourceData; + } + + // Adjust source stream + sourceData.pData += candidateData.size; + sourceData.size -= candidateData.size; + + dictionaryData.size = AddDictionary(candidateData, dictionaryData.size); + + if (candidateData.size > 3) + { + // Emit a dictionary reference + pDest += EmitReference(pDest, candidateData.pData - pSource, candidateData); + } + else if (bLastEmitIsLiteral) + { + // Concatenate this literal onto the previous literal + pDest += ConcatLiteral(pLastLiteralDest, candidateData); + bLastEmitIsLiteral = false; + } + else + { + // Emit a new literal + pLastLiteralDest = pDest; + bLastEmitIsLiteral = true; + pDest += EmitLiteral(pDest, candidateData); + } + } + + return pDest - pOriginalDest; +} + + +// +// This works, but it's stupidly slow, because it uses brute force, and +// because the brute force starts over everytime I grow the data string +// +int OLD_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) { printf("LZB_Compress %d bytes\n", sourceSize); @@ -45,6 +117,9 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) candidate_data.pData = pSource; candidate_data.size = 0; + int MatchOffset = -1; + int PreviousMatchOffset = -1; + while (processedBytes < sourceSize) { // Add a byte to the candidate_data, also tally number of processed @@ -60,11 +135,13 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) // (KMP is probably the last planned optmization here) + PreviousMatchOffset = MatchOffset; + MatchOffset = DictionaryMatch(candidate_data, bytesInDictionary); // The dictionary only contains bytes that have been emitted, so we // can't add this byte until we've emitted it? - if (DictionaryMatch(candidate_data, bytesInDictionary) < 0) + if (MatchOffset < 0) { // Was there a dictionary match @@ -72,7 +149,7 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) // this is an opportunity to use an assert candidate_data.size--; - int MatchOffset = DictionaryMatch(candidate_data, bytesInDictionary); + MatchOffset = PreviousMatchOffset; //DictionaryMatch(candidate_data, bytesInDictionary); if ((MatchOffset >= 0) && candidate_data.size > 3) { @@ -149,6 +226,25 @@ static int AddDictionary(const DataString& data, int dictionarySize) return dictionarySize; } + +DataString LongestMatch(const DataString& data, const DataString& dictionary) +{ + DataString result; + result.pData = nullptr; + result.size = 0; + + // Find the longest matching data in the dictionary + if (dictionary.size > 0) + { + // First Check for a pattern / run-length style match + + + } + + return result; +} + + //------------------------------------------------------------------------------ // // Return offset into dictionary where the string matches @@ -171,7 +267,9 @@ static int DictionaryMatch(const DataString& data, int dictionarySize) { // Check for pattern sizes, start small - int max_pattern_size = dictionarySize; + int max_pattern_size = 256; + if (dictionarySize < max_pattern_size) max_pattern_size = dictionarySize; + if (data.size < max_pattern_size) max_pattern_size = data.size; for (int pattern_size = 1; pattern_size <= max_pattern_size; ++pattern_size) { @@ -195,6 +293,10 @@ static int DictionaryMatch(const DataString& data, int dictionarySize) } } + // As an optimization + dictionarySize -= 1; // This last string has already been checked by, the + // run-length matcher above + if (dictionarySize < data.size) { return -1; From 955e8cf1d551350f23f21cb03e9c60c19b971ab0 Mon Sep 17 00:00:00 2001 From: dwsJason Date: Wed, 15 Jul 2020 23:59:33 -0400 Subject: [PATCH 08/21] faster string in string, WIP --- source/lzb.cpp | 95 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 85 insertions(+), 10 deletions(-) diff --git a/source/lzb.cpp b/source/lzb.cpp index 0e46214..281b02c 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -27,7 +27,7 @@ static int EmitReference(unsigned char *pDest, int dictionaryOffset, DataString& static int DictionaryMatch(const DataString& data, int dictionarySize); // Stuff I need for a faster version -static DataString LongestMatch(const DataString& data, const DataString* dictionary); +static DataString LongestMatch(const DataString& data, const DataString& dictionary); // // New Version, still Brute Force, but not as many times @@ -47,7 +47,7 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) sourceData.size = sourceSize; // Remember, this eventually will point at the frame buffer - dictionaryData.pData = pSource + dictionaryData.pData = pSource; dictionaryData.size = 0; // dumb last emit is a literal stuff @@ -61,7 +61,7 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) if (0 == candidateData.size) { candidateData.size = 1; - candidateData.pData = sourceData; + candidateData.pData = sourceData.pData; } // Adjust source stream @@ -73,13 +73,13 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) if (candidateData.size > 3) { // Emit a dictionary reference - pDest += EmitReference(pDest, candidateData.pData - pSource, candidateData); + pDest += (int)EmitReference(pDest, (int)(candidateData.pData - dictionaryData.pData), candidateData); + bLastEmitIsLiteral = false; } else if (bLastEmitIsLiteral) { // Concatenate this literal onto the previous literal pDest += ConcatLiteral(pLastLiteralDest, candidateData); - bLastEmitIsLiteral = false; } else { @@ -90,7 +90,7 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) } } - return pDest - pOriginalDest; + return (int)(pDest - pOriginalDest); } @@ -98,7 +98,7 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) // This works, but it's stupidly slow, because it uses brute force, and // because the brute force starts over everytime I grow the data string // -int OLD_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) +int old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) { printf("LZB_Compress %d bytes\n", sourceSize); @@ -226,6 +226,10 @@ static int AddDictionary(const DataString& data, int dictionarySize) return dictionarySize; } +//------------------------------------------------------------------------------ +// +// Return longest match of data, in dictionary +// DataString LongestMatch(const DataString& data, const DataString& dictionary) { @@ -234,11 +238,84 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary) result.size = 0; // Find the longest matching data in the dictionary - if (dictionary.size > 0) + if ((dictionary.size > 0) && (data.size > 0)) { + DataString candidate; + candidate.pData = data.pData; + candidate.size = 0; + // First Check for a pattern / run-length style match + // Check the end of the dictionary, to see if this data could be a + // pattern "run" (where we can repeat a pattern for X many times for free + // using the memcpy with overlapping source/dest buffers) + // (This is a dictionary based pattern run/length) + { + // Check for pattern sizes, start small + int max_pattern_size = 4096; + if (dictionary.size < max_pattern_size) max_pattern_size = dictionary.size; + if (data.size < max_pattern_size) max_pattern_size = data.size; + for (int pattern_size = 1; pattern_size <= max_pattern_size; ++pattern_size) + { + bool bMatch = true; + int pattern_start = dictionary.size - pattern_size; + for (int dataIndex = 0; dataIndex < data.size; ++dataIndex) + { + if (data.pData[ dataIndex ] == dictionary.pData[ pattern_start + (dataIndex % pattern_size) ]) + { + candidate.pData = dictionary.pData + pattern_start; + candidate.size = dataIndex+1; + continue; + } + + bMatch = false; + break; + } + + if (candidate.size < pattern_size) + break; + + if (candidate.size > result.size) + { + result = candidate; + } + } + } + + // As an optimization + int dictionarySize = dictionary.size - 1; // This last string has already been checked by, the + // run-length matcher above + + if (dictionarySize >= candidate.size) + { + // Check the dictionary for a match, brute force + for (int idx = 0; idx <= (dictionarySize-candidate.size); ++idx) + { + bool bMatch = true; + for (int dataIdx = 0; dataIdx < data.size; ++dataIdx) + { + if (data.pData[ dataIdx ] == dictionary.pData[ idx + dataIdx ]) + { + if (dataIdx > (candidate.size-1)) + { + candidate.pData = dictionary.pData + idx; + candidate.size = dataIdx - 1; + } + continue; + } + + bMatch = false; + break; + } + + if (candidate.size > result.size) + { + result = candidate; + break; + } + } + } } return result; @@ -302,8 +379,6 @@ static int DictionaryMatch(const DataString& data, int dictionarySize) return -1; } - int dictionaryOffset = 0; - int result = -1; // Check the dictionary for a match, brute force From f80008ca2110e681f1d9b27fb7e32af4990f0d45 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Thu, 16 Jul 2020 19:05:36 -0400 Subject: [PATCH 09/21] faster compressor: WIP --- source/lzb.cpp | 54 +++++++++++++++++++++++++++++++++++-------------- source/lzb.h | 1 + source/main.cpp | 6 ++++++ 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/source/lzb.cpp b/source/lzb.cpp index 281b02c..87212ef 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -9,6 +9,7 @@ #include "bctypes.h" #define MAX_DICTIONARY_SIZE (32 * 1024) +#define MAX_STRING_SIZE (16383) // // Yes This is a 32K Buffer, of bytes, with no structure to it // @@ -98,7 +99,7 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) // This works, but it's stupidly slow, because it uses brute force, and // because the brute force starts over everytime I grow the data string // -int old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) +int Old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) { printf("LZB_Compress %d bytes\n", sourceSize); @@ -257,7 +258,6 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary) for (int pattern_size = 1; pattern_size <= max_pattern_size; ++pattern_size) { - bool bMatch = true; int pattern_start = dictionary.size - pattern_size; for (int dataIndex = 0; dataIndex < data.size; ++dataIndex) @@ -269,12 +269,11 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary) continue; } - bMatch = false; break; } - if (candidate.size < pattern_size) - break; + //if (candidate.size < pattern_size) + // break; if (candidate.size > result.size) { @@ -284,34 +283,42 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary) } // As an optimization - int dictionarySize = dictionary.size - 1; // This last string has already been checked by, the + int dictionarySize = dictionary.size; // - 1; // This last string has already been checked by, the // run-length matcher above - if (dictionarySize >= candidate.size) + // As the size grows, we're missing potential matches in here + // I think the best way to counter this is to attempt somthing + // like KMP + + if (dictionarySize > candidate.size) { // Check the dictionary for a match, brute force - for (int idx = 0; idx <= (dictionarySize-candidate.size); ++idx) + for (int dictionaryIndex = 0; dictionaryIndex <= (dictionarySize-candidate.size); ++dictionaryIndex) { - bool bMatch = true; - for (int dataIdx = 0; dataIdx < data.size; ++dataIdx) + int sizeAvailable = dictionarySize - dictionaryIndex; + + if (sizeAvailable > data.size) sizeAvailable = data.size; + + // this could index off the end of the dictionary!!! FIX ME + for (int dataIndex = 0; dataIndex < sizeAvailable; ++dataIndex) { - if (data.pData[ dataIdx ] == dictionary.pData[ idx + dataIdx ]) + if (data.pData[ dataIndex ] == dictionary.pData[ dictionaryIndex + dataIndex ]) { - if (dataIdx > (candidate.size-1)) + if (dataIndex >= candidate.size) { - candidate.pData = dictionary.pData + idx; - candidate.size = dataIdx - 1; + candidate.pData = dictionary.pData + dictionaryIndex; + candidate.size = dataIndex + 1; } continue; } - bMatch = false; break; } if (candidate.size > result.size) { result = candidate; + //dictionaryIndex = -1; break; } } @@ -504,11 +511,16 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) { int decompressedBytes = 0; + unsigned char *pOriginalSource = pSource; + while (decompressedBytes < destSize) { u16 opcode = *pSource++; opcode |= ((u16)(*pSource++))<<8; + //printf("%04X:", (unsigned int)(pSource-pOriginalSource)); + + if (opcode & 0x8000) { // Dictionary @@ -518,8 +530,18 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) u16 offset = *pSource++; offset |= ((u16)(*pSource++))<<8; + const char* overlapped = ""; + + if ((&pDest[ decompressedBytes ] - &pDest[ offset ]) < opcode) + { + overlapped = "pattern"; + } + my_memcpy(&pDest[ decompressedBytes ], &pDest[ offset ], opcode); decompressedBytes += opcode; + + + //printf("%04X:Dic %04X %s\n",decompressedBytes, (unsigned int)opcode, overlapped); } else { @@ -527,6 +549,8 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) memcpy(&pDest[ decompressedBytes ], pSource, opcode); decompressedBytes += opcode; pSource += opcode; + + //printf("%04X:Lit %04X\n",decompressedBytes, (unsigned int)opcode); } } } diff --git a/source/lzb.h b/source/lzb.h index 8284aa2..015ba4b 100644 --- a/source/lzb.h +++ b/source/lzb.h @@ -8,6 +8,7 @@ // returns the size of data saved into the pDest Buffer // int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize); +int Old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize); void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize); #endif // LZB_H diff --git a/source/main.cpp b/source/main.cpp index fcb0d00..f62b9d4 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -114,14 +114,20 @@ int main(int argc, char* argv[]) const std::vector& c1Datas = c2data.GetPixelMaps(); unsigned char workbuffer[64*1024]; + unsigned char workbuffer2[64*1024]; for (int idx = 0; idx < frameCount; ++idx) { + int oldCompressedSize = Old_LZB_Compress(workbuffer2, c1Datas[ idx ], 32 * 1024); + printf("old compressedSize = %d\n", oldCompressedSize); int compressedSize = LZB_Compress(workbuffer, c1Datas[ idx ], 32 * 1024); printf("compressedSize = %d\n", compressedSize); unsigned char validationBuffer[ 32 * 1024 ]; + printf("Decompress OLD\n"); + LZB_Decompress(validationBuffer, workbuffer2, 32 * 1024); + printf("Decompress NEW\n"); LZB_Decompress(validationBuffer, workbuffer, 32 * 1024); if (0 == memcmp(c1Datas[ idx ], validationBuffer, 32*1024)) From 3a7b5a69fb6bc188d1a41de8fa50208b7c866427 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Fri, 17 Jul 2020 14:33:32 -0400 Subject: [PATCH 10/21] These compressors are "good enough" --- source/lzb.cpp | 28 +++++++++++++++++++++++----- source/main.cpp | 20 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/source/lzb.cpp b/source/lzb.cpp index 87212ef..72c17c3 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -59,7 +59,9 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) { candidateData = LongestMatch(sourceData, dictionaryData); - if (0 == candidateData.size) + // If no match, or the match is too small, then take the next byte + // and emit as literal + if ((0 == candidateData.size)) // || (candidateData.size < 4)) { candidateData.size = 1; candidateData.pData = sourceData.pData; @@ -161,7 +163,22 @@ int Old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSiz } else { - candidate_data.size++; + if (0 == candidate_data.size) + { + candidate_data.size++; + } + else + { + processedBytes--; + + //if (candidate_data.size > 1) + //{ + // processedBytes -= (candidate_data.size - 1); + // candidate_data.size = 1; + //} + } + + // Add Dictionary bytesInDictionary = AddDictionary(candidate_data, bytesInDictionary); @@ -177,6 +194,7 @@ int Old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSiz bytesEmitted += EmitLiteral(pDest + bytesEmitted, candidate_data); } bLastEmitIsLiteral = true; + //MatchOffset = -1; } } } @@ -518,7 +536,7 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) u16 opcode = *pSource++; opcode |= ((u16)(*pSource++))<<8; - //printf("%04X:", (unsigned int)(pSource-pOriginalSource)); +// printf("%04X:", (unsigned int)(pSource-pOriginalSource)); if (opcode & 0x8000) @@ -541,7 +559,7 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) decompressedBytes += opcode; - //printf("%04X:Dic %04X %s\n",decompressedBytes, (unsigned int)opcode, overlapped); +// printf("%04X:Dic %04X %s\n",decompressedBytes, (unsigned int)opcode, overlapped); } else { @@ -550,7 +568,7 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) decompressedBytes += opcode; pSource += opcode; - //printf("%04X:Lit %04X\n",decompressedBytes, (unsigned int)opcode); +// printf("%04X:Lit %04X\n",decompressedBytes, (unsigned int)opcode); } } } diff --git a/source/main.cpp b/source/main.cpp index f62b9d4..a7325d1 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -113,6 +113,25 @@ int main(int argc, char* argv[]) const std::vector& c1Datas = c2data.GetPixelMaps(); + #if 1 + unsigned char workbuffer[64*1024]; + for (int idx = 0; idx < frameCount; ++idx) + { + int compressedSize = LZB_Compress(workbuffer, c1Datas[ idx ], 32 * 1024); + printf("compressedSize = %d\n", compressedSize); + unsigned char validationBuffer[ 32 * 1024 ]; + LZB_Decompress(validationBuffer, workbuffer, 32 * 1024); + if (0 == memcmp(c1Datas[ idx ], validationBuffer, 32*1024)) + { + printf("Decompression Validated\n"); + } + else + { + printf("Decompression Corrupted\n"); + } + } + + #else unsigned char workbuffer[64*1024]; unsigned char workbuffer2[64*1024]; @@ -140,6 +159,7 @@ int main(int argc, char* argv[]) } } + #endif } } From 89f6eb72e52f970c42c05ddaf4e915a484881987 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Sat, 18 Jul 2020 12:19:51 -0400 Subject: [PATCH 11/21] WIP on the GSLA file serializer --- source/gsla_file.cpp | 272 +++++++++++++++++++++++++++++++++++ source/gsla_file.h | 143 ++++++++++++++++++ vcxproj/gsla.vcxproj | 2 + vcxproj/gsla.vcxproj.filters | 6 + 4 files changed, 423 insertions(+) create mode 100644 source/gsla_file.cpp create mode 100644 source/gsla_file.h diff --git a/source/gsla_file.cpp b/source/gsla_file.cpp new file mode 100644 index 0000000..5f90fad --- /dev/null +++ b/source/gsla_file.cpp @@ -0,0 +1,272 @@ +// +// C++ Encoder/Decoder +// For GSLA, GS Lzb Animation File Format +// +// +// Care is taken in the encoder, to make sure the 65816 does not have to cross +// bank boundaries during any copy. This is so we can use the MVN instruction, +// and so we can reduce the number of bank checks in the code. We will have an +// opcode, that says “source data bank has changed” +// +// The file will be laid out such that you load the file in at a 64K memory +// boundary +// +// Goals include a good balance between file size, and playback performance +// (since one often makes a trade off with the other). +// +// The file is defined as a byte stream, loaded on a 64K Bank Boundary +// +// +// file-offset: the thing that is at the offset +// +// Header of the File is 20 bytes as follows +// +//File Offset Data Commentary +//------------------------------------------------------------------ +//0 0x47 ; ‘G’ Graphics +//1 0x53 ; ‘S’ +//2 0x4C ; ‘L’ LZB +//3 0x41 ; ‘A’ Animation +// +// File Length, is the total length of the file +//4 FileLengthLow ; Low byte, 32-bit file length +//5 LengthLowHigh ; High byte of low word +//6 LengthHighLow ; Low byte, of high word +//7 LengthHighHigh ; High Byte, of high word +// +// 16 bit word with version # +//8 VL ; Version # of the file format, currently only version 0 exists +//9 VH ; Version High Byte +// ; %RVVV_VVVV_VVVV_VVVV +// ; V is a version #, 0 for now +// ; R is the MSB, R = 0 no ring frame +// ; R = 1, there is a ring frame +// ; A Ring Frame is a frame that will delta from the last +// ; frame of the animation, back to the first, for smoother +// ; playback looping , If a ring frame exists, it’s also +// ; included in the frame count +// +// next is a word, width in bytes (likely 160 for now) +//0xA WL ; Display Width in bytes low byte +//0xB WH ; Display Width in bytes high byte +// +// next is a word, height (likely 200 for now) +//0xC HL ; Display Height in bytes, low byte +//0xD HH ; Display Height in bytes, high byte +// 2 bytes, Frame Size in Bytes, since a “Frame” may contain more than just the +// width * height, worth of pixels, for now this is $8000, or 32768 +//0xE FBL ; Frame Buffer Length Low +//0xF FBH ; Frame Buffer Length High +// +// 4 byte, 32-bit, Frame Count (includes total frame count, so if there is a ring frame, this is included in the total) +//0x10 FrameCountLow +//0x11 FrameCountLowHigh +//0x12 FrameCountHighLow +//0x13 FrameCountHigh +// +// +// After this comes AIFF style chunks of data, basically a 4 byte chunk name, +// followed by a 4 byte length (inclusive of the chunk size). The idea is that +// you can skip chunks you don’t understand. +// +//File Offset: +//0x14 First Chunk (followed by more Chunks, until end of file) +// +//Chunk Definitions +//Name: ‘INIT’ - Initial Frame Chunk, this is the data used to first initialize the playback buffer +//0: 0x49 ‘I’ +//1: 0x4E ‘N’ +//2: 0x49 ‘I’ +//3: 0x54 ‘T’ +// 32 bit long, length, little endian, including the 8 byte header +//4: length low low +//5: length low high +//6: length high low +//7: length high high +// +//8: …. This is a single frame of data, that decodes/decompresses into frame +// sized bytes (right now 0x8000) +// This data stream includes, an end of animation opcode, so that the normal +// animation decompressor, can be called on this data, and it will emit the +// initial frame onto the screen +// +//Name: ‘ANIM’ - Frames +//0: 0x41 ‘A’ +//1: 0x4E ‘N’ +//2: 0x49 ‘I’ +//3: 0x4D ‘M’ +// 32 bit long, length, little endian, including chunk header +//4: length low low +//5: length low high +//6: length high low +//7: length high high +// +// This is followed by the frames, with the intention of decompressing them at +// 60FPS, which is why no play speed is included, if you need a play-rate +// slower than this, blank frame’s should be inserted into the animation data +// +// Every attempt is made to delta encode the image, meaning we just encode +// information about what changed each frame. We attempt to make the size +// efficient by supporting dictionary copies (where the dictionary is made up +// of existing pixels in the frame buffer). +// +//Command Word, encoded low-high, what the bits mean: +// +// xxx_xxxx_xxxx_xxx is the number of bytes 1-16384 to follow (0 == 1 byte) +// +//%0xxx_xxxx_xxxx_xxx0 - Copy Bytes - straight copy bytes +//%1xxx_xxxx_xxxx_xxx1 - Skip Bytes - skip bytes / move the cursor +//%1xxx_xxxx_xxxx_xxx0 - Dictionary Copy Bytes from frame buffer to frame buffer +// +//%0000_0000_0000_0001- Source Skip -> Source pointer skips to next bank of data +//%0000_0000_0000_0011- End of Frame - end of frame +//%0000_0000_0000_0111- End of Animation / End of File / no more frames +// +// other remaining codes, are reserved for future expansion + +#include "gsla_file.h" +#include + +// If these structs are the wrong size, there's an issue with type sizes, and +// your compiler +static_assert(sizeof(GSLA_Header)==20, "GSLA_Header is supposed to be 20 bytes"); +static_assert(sizeof(GSLA_INIT)==8, "GSLA_INIT is supposed to be 8 bytes"); +static_assert(sizeof(GSLA_ANIM)==8, "GSLA_ANIM is supposed to be 8 bytes"); +static_assert(sizeof(GSLA_CHUNK)==8, "GSLA_CHUNK is supposed to be 8 bytes"); + +//------------------------------------------------------------------------------ +// Load in a FanFile constructor +// +GSLAFile::GSLAFile(const char *pFilePath) + : m_widthPixels(320) + , m_heightPixels(200) +{ + LoadFromFile(pFilePath); +} +//------------------------------------------------------------------------------ + +GSLAFile::~GSLAFile() +{ + // Free Up the memory + for (int idx = 0; idx < m_pC1PixelMaps.size(); ++idx) + { + delete[] m_pC1PixelMaps[idx]; + m_pC1PixelMaps[ idx ] = nullptr; + } +} + +//------------------------------------------------------------------------------ + +void GSLAFile::LoadFromFile(const char* pFilePath) +{ + // Free Up the memory + for (int idx = 0; idx < m_pC1PixelMaps.size(); ++idx) + { + delete[] m_pC1PixelMaps[idx]; + m_pC1PixelMaps[ idx ] = nullptr; + } + + m_pC1PixelMaps.clear(); + //-------------------------------------------------------------------------- + + std::vector bytes; + + //-------------------------------------------------------------------------- + // Read the file into memory + FILE* pFile = nullptr; + errno_t err = fopen_s(&pFile, pFilePath, "rb"); + + if (0==err) + { + fseek(pFile, 0, SEEK_END); + size_t length = ftell(pFile); // get file size + fseek(pFile, 0, SEEK_SET); + + bytes.resize( length ); // make sure buffer is large enough + + // Read in the file + fread(&bytes[0], sizeof(unsigned char), bytes.size(), pFile); + fclose(pFile); + } + + if (bytes.size()) + { + size_t file_offset = 0; // File Cursor + + // Bytes are in the buffer, so let's start looking at what we have + GSLA_Header* pHeader = (GSLA_Header*) &bytes[0]; + + // Early out if things don't look right + if (!pHeader->IsValid((unsigned int)bytes.size())) + return; + + // Size in bytes for each frame in this animation + m_frameSize = pHeader->frame_size; + + // pre-allocate all the frames + for (unsigned int idx = 0; idx < pHeader->frame_count; ++idx) + { + m_pC1PixelMaps.push_back(new unsigned char[ m_frameSize ]); + } + + //---------------------------------------------------------------------- + // Process Chunks as we encounter them + file_offset += sizeof(GSLA_Header); + + // While we're not at the end of the file + while (file_offset < bytes.size()) + { + // This is pretty dumb, just get it done + // These are the types I understand + // every chunk is supposed to contain a value chunk_length + // at offset +4, so that we can ignore ones we don't understand + GSLA_INIT* pINIT = (GSLA_INIT*)&bytes[ file_offset ]; + GSLA_ANIM* pANIM = (GSLA_ANIM*)&bytes[ file_offset ]; + GSLA_CHUNK* pCHUNK = (GSLA_CHUNK*)&bytes[ file_offset ]; + + if (pINIT->IsValid()) + { + // We have an initial frame chunk + UnpackInitialFrame(pINIT); + } + else if (pANIM->IsValid()) + { + // We have a packed animation frames chunk + UnpackAnimation(pANIM, pHeader); + } + + file_offset += pCHUNK->chunk_length; + + } + } +} + +//------------------------------------------------------------------------------ + +//------------------------------------------------------------------------------ +// +// Unpack the initial frame, that's been packed with an empty initial dictionary +// So every byte of the buffer will be written out (no skip opcodes) +// +void GSLAFile::UnpackInitialFrame(GSLA_INIT* pINIT) +{ + unsigned char *pData = ((unsigned char*)pINIT) + sizeof(GSLA_INIT); + + unsigned char *pTargetBuffer = m_pC1PixelMaps[ 0 ]; // Data needs to be pre allocated + + //DecompressFrame(pTargetBuffer, pData); +} + +//------------------------------------------------------------------------------ +// +// Unpack the animation frame, assuming that the initial frame already exists +// +void GSLAFile::UnpackAnimation(GSLA_ANIM* pANIM, GSLA_Header* pHeader) +{ + unsigned char *pData = ((unsigned char*)pANIM) + sizeof(GSLA_ANIM); + + unsigned char *pTargetBuffer = m_pC1PixelMaps[ 0 ]; // Data needs to be pre allocated + + //DecompressAnim(pTargetBuffer, pData); +} + diff --git a/source/gsla_file.h b/source/gsla_file.h new file mode 100644 index 0000000..3c4e597 --- /dev/null +++ b/source/gsla_file.h @@ -0,0 +1,143 @@ +// +// C++ Encoder/Decoder +// For GSLA, GS Lzb Animation File Format +// +// +// Care is taken in the encoder, to make sure the 65816 does not have to cross +// bank boundaries during any copy. This is so we can use the MVN instruction, +// and so we can reduce the number of bank checks in the code. We will have an +// opcode, that says “source data bank has changed” +// +// The file will be laid out such that you load the file in at a 64K memory +// boundary +// +// Please see the gsla_file.cpp, for documentation on the actual +// format of the file +// +#ifndef GSLA_FILE_H +#define GSLA_FILE_H + +#include + +#pragma pack(push, 1) + +// Data Stream Header +typedef struct GSLA_Header +{ + char G,S,L,A; + unsigned int file_length; // length of the file, including this header + unsigned short version; + unsigned short width; // data width in bytes + unsigned short height; // data height in bytes + unsigned short frame_size; // frame size in bytes (0 means 64K) + unsigned int frame_count; // total number of frames in the file + +//------------------------------------------------------------------------------ +// If you're doing C, just get rid of these methods + bool IsValid(unsigned int fileLength) + { + if ((G!='G')||(S!='S')||(L!='L')||(A!='A')) + return false; // signature is not right + + if (file_length != fileLength) + return false; // size isn't right + + if (0x8000 != frame_size) + return false; + + if (160 != width) + return false; + + if (200 != height) + return false; + + return true; + } + +} GSLA_Header; + +// INITial Frame Chunk +typedef struct GSLA_INIT +{ + char I,N,i,T; // 'I','N','I','T' + unsigned int chunk_length; // in bytes, including the 9 bytes header of this chunk + + // Commands Coded Data Follows +//------------------------------------------------------------------------------ +// If you're doing C, just get rid of these methods + bool IsValid() + { + if ((I!='I')||(N!='N')||(i!='I')||(T!='T')) + return false; // signature is not right + + return true; + } + + +} GSLA_INIT; + +// Animated Frames Chunk +typedef struct GSLA_ANIM +{ + char A,N,I,M; // 'A','N','I','M' + unsigned int chunk_length; // in bytes, including the 8 bytes header of this chunk + + // Commands Coded Data Follows + +//------------------------------------------------------------------------------ +// If you're doing C, just get rid of these methods + bool IsValid() + { + if ((A!='A')||(N!='N')||(I!='I')||(M!='M')) + return false; // signature is not right + + return true; + } + +} GSLA_ANIM; + +// Generic Unknown Chunk +typedef struct GSLA_CHUNK +{ + char id0,id1,id2,id3; + unsigned int chunk_length; + +} GSLA_CHUNK; + + +#pragma pack(pop) + +class GSLAFile +{ +public: + // Load in a C2 File + GSLAFile(const char *pFilePath); + + ~GSLAFile(); + + // Retrieval + void LoadFromFile(const char* pFilePath); + int GetFrameCount() { return (int)m_pC1PixelMaps.size(); } + int GetWidth() { return m_widthPixels; } + int GetHeight() { return m_heightPixels; } + + const std::vector& GetPixelMaps() { return m_pC1PixelMaps; } + +private: + + void UnpackInitialFrame(GSLA_INIT* pINIT); + void UnpackAnimation(GSLA_ANIM* pANIM, GSLA_Header* pHeader); + + + int m_frameSize; // frame buffer size in bytes + + int m_widthPixels; // Width of image in pixels + int m_heightPixels; // Height of image in pixels + + std::vector m_pC1PixelMaps; + +}; + + +#endif // C2_FILE_H + diff --git a/vcxproj/gsla.vcxproj b/vcxproj/gsla.vcxproj index 3e0d09f..24ac719 100644 --- a/vcxproj/gsla.vcxproj +++ b/vcxproj/gsla.vcxproj @@ -141,10 +141,12 @@ + + diff --git a/vcxproj/gsla.vcxproj.filters b/vcxproj/gsla.vcxproj.filters index 3ab00b7..70b137e 100644 --- a/vcxproj/gsla.vcxproj.filters +++ b/vcxproj/gsla.vcxproj.filters @@ -20,6 +20,9 @@ Header Files + + Source Files + @@ -31,5 +34,8 @@ Source Files + + Source Files + \ No newline at end of file From 7712398e0c6cf25db29bb6b228399e5ca82bbe08 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Sat, 18 Jul 2020 15:49:27 -0400 Subject: [PATCH 12/21] GSLA encoder: WIP --- source/gsla_file.cpp | 197 ++++++++++++++++++++++++++++++++++++++++++- source/gsla_file.h | 6 +- source/lzb.cpp | 16 ++++ source/lzb.h | 6 ++ source/main.cpp | 28 +++++- 5 files changed, 246 insertions(+), 7 deletions(-) diff --git a/source/gsla_file.cpp b/source/gsla_file.cpp index 5f90fad..159274c 100644 --- a/source/gsla_file.cpp +++ b/source/gsla_file.cpp @@ -114,17 +114,21 @@ // // xxx_xxxx_xxxx_xxx is the number of bytes 1-16384 to follow (0 == 1 byte) // -//%0xxx_xxxx_xxxx_xxx0 - Copy Bytes - straight copy bytes +//%0xxx_xxxx_xxxx_xxx1 - Copy Bytes - straight copy bytes //%1xxx_xxxx_xxxx_xxx1 - Skip Bytes - skip bytes / move the cursor //%1xxx_xxxx_xxxx_xxx0 - Dictionary Copy Bytes from frame buffer to frame buffer // -//%0000_0000_0000_0001- Source Skip -> Source pointer skips to next bank of data -//%0000_0000_0000_0011- End of Frame - end of frame -//%0000_0000_0000_0111- End of Animation / End of File / no more frames +//%0000_0000_0000_0000- Source Skip -> Source pointer skips to next bank of data +//%0000_0000_0000_0010- End of Frame - end of frame +//%0000_0000_0000_0110- End of Animation / End of File / no more frames +// // // other remaining codes, are reserved for future expansion #include "gsla_file.h" + +#include "lzb.h" + #include // If these structs are the wrong size, there's an issue with type sizes, and @@ -143,6 +147,17 @@ GSLAFile::GSLAFile(const char *pFilePath) { LoadFromFile(pFilePath); } + +//------------------------------------------------------------------------------ + +GSLAFile::GSLAFile(int iWidthPixels, int iHeightPixels, int iFrameSizeBytes ) + : m_widthPixels(iWidthPixels) + , m_heightPixels(iHeightPixels) + , m_frameSize( iFrameSizeBytes ) +{ + +} + //------------------------------------------------------------------------------ GSLAFile::~GSLAFile() @@ -270,3 +285,177 @@ void GSLAFile::UnpackAnimation(GSLA_ANIM* pANIM, GSLA_Header* pHeader) //DecompressAnim(pTargetBuffer, pData); } +//------------------------------------------------------------------------------ +// +// Append a copy of raw image data into the class +// +void GSLA::AddImages( const std::vector& pFrameBytes ) +{ + for (int idx = 0; idx < pFrameBytes.size(); ++idx) + { + unsigned char* pPixels = new unsigned char[ m_frameSize ]; + memcpy(pPixels, pFrameBytes[ idx ], m_frameSize ); + m_pC1PixelMaps.push_back( pPixels ); + } +} + +//------------------------------------------------------------------------------ +// +// Compress / Serialize a new GSLA File +// +void GSLA::SaveToFile(const char* pFilenamePath) +{ + // We're not going to even try encoding an empty file + if (m_pC1PixelMaps.size() < 1) + { + return; + } + + // serialize to memory, then save that to a file + std::vector bytes; + + //-------------------------------------------------------------------------- + // Add the header + bytes.resize( bytes.size() + sizeof(GSLA_Header) ); + + //$$JGA Rememeber, you have to set the pointer, before every access + //$$JGA to the header data, because vector is going to change out + //$$JGA memory addresses from underneath you + GSLA_Header* pHeader = (GSLA_Header*)&bytes[0]; + + pHeader->G = 'G'; pHeader->S = 'S'; pHeader->L = 'L'; pHeader->A = 'A'; + + pHeader->file_length = 0; // Temp File Length + + pHeader->version = 0x8000; // Version 0, with a Ring/Loop Frame at the end + + pHeader->width = m_widthPixels >> 1; + pHeader->height = m_widthHeight; + + pHeader->frame_size = m_frameSize; + + pHeader->frame_count = m_pC1PixelMaps.size() + 1; // + 1 for the ring frame + + //-------------------------------------------------------------------------- + // Add the INITial frame chunk + // + // If there's only an initial frame, I guess this becomes a picture + // + + size_t init_offset = bytes.size(); + + // Add space for the INIT header + bytes.resize( bytes.size() + sizeof(GSLA_INIT) ); + GSLA_INIT* pINIT = (GSLA_INIT*) &bytes[ init_offset ]; + + pINIT->I = 'I'; pINIT->N = 'N'; pINIT->i = 'I'; pINIT->T = 'T'; + pINIT->chunk_length = 0; // temp chunk size + + // Need a place to put compressed data, in theory it could be bigger + // than the original data, I think if that happens, the image was probably + // designed to break this, anyway, give double theoretical max + unsigned char* pWorkBuffer = new unsigned char[ m_frameSize * 2 ]; + + unsigned char* pInitialFrame = m_pC1PixelMaps[ 0 ]; + + // We're not worried about bank wrap on the first frame, and we don't have a pre-populated + // dictionary + int compressedSize = LZBA_Compress(pWorkBuffer, pInitialFrame, m_frameSize, + pInitialFrame, pInitialFrame); + + for (int compressedIndex = 0; compressedIndex < compressedSize; ++compressedIndex) + { + bytes.push_back(pWorkBuffer[ compressedIndex ]); + } + + // Insert EOF/ End of Animation Done opcode + bytes.push_back( 0x06 ); + bytes.push_back( 0x00 ); + + + // Reset pointer to the pINIT (as the baggage may have shifted) + pINIT = (GSLA_INIT*) &bytes[ init_offset ]; + pINIT->chunk_length = (unsigned int) (bytes.size() - init_offset); + + //-------------------------------------------------------------------------- + // Add the ANIMation frames chunk + // + // We always add this, because we always add a Ring/Loop frame, we always + // end up with at least 2 frames + // + + size_t anim_offset = bytes.size(); + + // Add Space for the ANIM Header + bytes.resize( bytes.size() + sizeof(GLSA_ANIM) ); + GSLA_ANIM* pANIM = (GSLA_ANIM*) &bytes[ anim_offset ]; + + pANIM->A = 'A'; pANIM->N = 'N'; pANIM->I ='I'; pANIM->M = 'M'; + pANIM->chunk_length = 0; // temporary chunk size + + // Initialize the Canvas with the initial frame (we alread exported this) + unsigned char *pCanvas = new unsigned char[ m_frameSize ]; + memcpy(pCanvas, m_pC1PixelMaps[0], m_frameSize); + + memcpy(pCanvas, m_pC1PixelMaps[0], m_frameSize); + + // Let's encode some frames buddy + for (int frameIndex = 1; frameIndex < m_pC1PixelMaps.size(); ++frameIndex) + { + // I don't want random data in the bank gaps, so initialize this + // buffer with zero + memset(pWorkBuffer, 0, m_frameSize * 2); + + int frameSize = LZBA_Compress(pWorkBuffer, m_pC1PixelMaps[ frameIndex], + m_frameSize, pWorkBuffer-bytes.size(), + pCanvas, m_frameSize ); + + for (int frameIndex = 0; frameIndex < frameSize; ++frameIndex) + { + bytes.push_back(pWorkBuffer[ frameIndex ]); + } + } + + // Add the RING Frame + memset(pWorkBuffer, 0, m_frameSize * 2); + + int ringSize = LZBA_Compress(pWorkBuffer, m_pC1PixelMaps[ 0 ], + m_frameSize, pWorkBuffer-bytes.size(), + pCanvas, m_frameSize ); + + for (int ringIndex = 0; ringIndex < ringSize; ++ringIndex) + { + bytes.push_back(pWorkBuffer[ ringIndex ]); + } + + delete[] pCanvas; pCanvas = nullptr; + + // Insert End of file/ End of Animation Done opcode + bytes.push_back( 0x06 ); + bytes.push_back( 0x00 ); + + // Update the chunk length + pFRAM = (FanFile_FRAM*)&bytes[ fram_offset ]; + pFRAM->chunk_length = (unsigned int) (bytes.size() - fram_offset); + + // Update the header + pHeader = (FanFile_Header*)&bytes[0]; // Required + pHeader->file_length = (unsigned int)bytes.size(); // get some valid data in there + + // Try not to leak memory, even though we probably do + delete[] pWorkBuffer; + + //-------------------------------------------------------------------------- + // Create the file and write it + FILE* pFile = nullptr; + errno_t err = fopen_s(&pFile, pFilenamePath, "wb"); + + if (0==err) + { + fwrite(&bytes[0], sizeof(unsigned char), bytes.size(), pFile); + fclose(pFile); + } +} + +//------------------------------------------------------------------------------ + diff --git a/source/gsla_file.h b/source/gsla_file.h index 3c4e597..70bf6aa 100644 --- a/source/gsla_file.h +++ b/source/gsla_file.h @@ -112,9 +112,13 @@ class GSLAFile public: // Load in a C2 File GSLAFile(const char *pFilePath); - ~GSLAFile(); + // Creation + GSLAFile(int iWidthPixels, int iHeightPixels, int iFrameSizeBytes); + void AddImages( const std::vector& pFrameBytes ); + void SaveToFile(const char* pFilenamePath); + // Retrieval void LoadFromFile(const char* pFilePath); int GetFrameCount() { return (int)m_pC1PixelMaps.size(); } diff --git a/source/lzb.cpp b/source/lzb.cpp index 72c17c3..544aa2e 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -574,4 +574,20 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) } //------------------------------------------------------------------------------ +// +// Encode a Frame in GSLA LZB Format +// +int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, unsigned char* pDataStart, int dictionarySize) +{ +} + +//------------------------------------------------------------------------------ +// +// Decompress a Frame in the GSLA LZB Format +// +int LZBA_Decompress(unsigned char* pDest, unsigned char* pSource, unsigned char* pDataStart) +{ +} + +//------------------------------------------------------------------------------ diff --git a/source/lzb.h b/source/lzb.h index 015ba4b..d8d624e 100644 --- a/source/lzb.h +++ b/source/lzb.h @@ -11,5 +11,11 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize); int Old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize); void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize); +// +// LZB Compressor that uses GSLA Opcodes while encoding +// +int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, unsigned char* pDataStart, unsigned char* pDictionary, int dictionarySize=0); +int LZBA_Decompress(unsigned char* pDest, unsigned char* pSource, unsigned char* pDataStart); + #endif // LZB_H diff --git a/source/main.cpp b/source/main.cpp index a7325d1..699dada 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -1,5 +1,7 @@ // // GSLA - GS LZB Animation Tool +// +// Look in gsla_file.cpp/h for more information about the file format // #include @@ -7,16 +9,20 @@ #include #include "c2_file.h" +#include "gsla_file.h" + #include "lzb.h" //------------------------------------------------------------------------------ static void helpText() { - printf("GSLA - v0.0\n"); + printf("GSLA - v1.0\n"); printf("--------------\n"); - printf("GS Lzb Animation Tool\n"); + printf("GS Lzb Animation Creation Tool\n"); printf("\n"); printf("\ngsla [options] \n"); + printf("\n\n There are no [options] yet\n"); + printf("Converts from C2 to GSLA\n"); exit(-1); } @@ -111,8 +117,25 @@ int main(int argc, char* argv[]) helpText(); } + if (pOutfilePath) + { + const std::vector& c1Datas = c2data.GetPixelMaps(); + + printf("Saving %s with %d frames\n", pOutfilePath, (int)c1Datas.size()); + + GSLAFile anim(320,200, 0x8000); + + anim.AddImages(c1Datas); + + anim.SaveToFile(pOutfilePath); + + } + + #if 0 const std::vector& c1Datas = c2data.GetPixelMaps(); + // LZB Testing Code, that I should probalby remove from this file + // Just ignore this block of code #if 1 unsigned char workbuffer[64*1024]; for (int idx = 0; idx < frameCount; ++idx) @@ -160,6 +183,7 @@ int main(int argc, char* argv[]) } #endif + #endif } } From bb916c15e251962e2d3c6d22a815e95514fe3bda Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Sat, 18 Jul 2020 16:41:01 -0400 Subject: [PATCH 13/21] GSLA: WIP --- source/gsla_file.cpp | 16 +++++----- source/lzb.cpp | 72 ++++++++++++++++++++++++++++++++++++++------ source/lzb.h | 4 ++- 3 files changed, 74 insertions(+), 18 deletions(-) diff --git a/source/gsla_file.cpp b/source/gsla_file.cpp index 159274c..3d9b8ce 100644 --- a/source/gsla_file.cpp +++ b/source/gsla_file.cpp @@ -289,7 +289,7 @@ void GSLAFile::UnpackAnimation(GSLA_ANIM* pANIM, GSLA_Header* pHeader) // // Append a copy of raw image data into the class // -void GSLA::AddImages( const std::vector& pFrameBytes ) +void GSLAFile::AddImages( const std::vector& pFrameBytes ) { for (int idx = 0; idx < pFrameBytes.size(); ++idx) { @@ -303,7 +303,7 @@ void GSLA::AddImages( const std::vector& pFrameBytes ) // // Compress / Serialize a new GSLA File // -void GSLA::SaveToFile(const char* pFilenamePath) +void GSLAFile::SaveToFile(const char* pFilenamePath) { // We're not going to even try encoding an empty file if (m_pC1PixelMaps.size() < 1) @@ -330,11 +330,11 @@ void GSLA::SaveToFile(const char* pFilenamePath) pHeader->version = 0x8000; // Version 0, with a Ring/Loop Frame at the end pHeader->width = m_widthPixels >> 1; - pHeader->height = m_widthHeight; + pHeader->height = m_heightPixels; pHeader->frame_size = m_frameSize; - pHeader->frame_count = m_pC1PixelMaps.size() + 1; // + 1 for the ring frame + pHeader->frame_count = (unsigned int)m_pC1PixelMaps.size() + 1; // + 1 for the ring frame //-------------------------------------------------------------------------- // Add the INITial frame chunk @@ -387,7 +387,7 @@ void GSLA::SaveToFile(const char* pFilenamePath) size_t anim_offset = bytes.size(); // Add Space for the ANIM Header - bytes.resize( bytes.size() + sizeof(GLSA_ANIM) ); + bytes.resize( bytes.size() + sizeof(GSLA_ANIM) ); GSLA_ANIM* pANIM = (GSLA_ANIM*) &bytes[ anim_offset ]; pANIM->A = 'A'; pANIM->N = 'N'; pANIM->I ='I'; pANIM->M = 'M'; @@ -435,11 +435,11 @@ void GSLA::SaveToFile(const char* pFilenamePath) bytes.push_back( 0x00 ); // Update the chunk length - pFRAM = (FanFile_FRAM*)&bytes[ fram_offset ]; - pFRAM->chunk_length = (unsigned int) (bytes.size() - fram_offset); + pANIM = (GSLA_ANIM*)&bytes[ anim_offset ]; + pANIM->chunk_length = (unsigned int) (bytes.size() - anim_offset); // Update the header - pHeader = (FanFile_Header*)&bytes[0]; // Required + pHeader = (GSLA_Header*)&bytes[0]; // Required pHeader->file_length = (unsigned int)bytes.size(); // get some valid data in there // Try not to leak memory, even though we probably do diff --git a/source/lzb.cpp b/source/lzb.cpp index 544aa2e..98caa18 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -575,18 +575,72 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) //------------------------------------------------------------------------------ // -// Encode a Frame in GSLA LZB Format +// Compress a Frame in the GSLA LZB Format // -int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, unsigned char* pDataStart, int dictionarySize) +int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, + unsigned char* pDataStart, unsigned char* pDictionary, + int dictionarySize) { -} +// printf("LZBA Compress %d bytes\n", sourceSize); + + unsigned char *pOriginalDest = pDest; + + DataString sourceData; + DataString dictionaryData; + DataString candidateData; + + // Source Data Stream - will compress until the size is zero + sourceData.pData = pSource; + sourceData.size = sourceSize; + + // Dictionary is the Frame Buffer + dictionaryData.pData = pDictionary; + dictionaryData.size = dictionarySize; + + // dumb last emit is a literal stuff + bool bLastEmitIsLiteral = false; + unsigned char* pLastLiteralDest = nullptr; + + while (sourceData.size > 0) + { + candidateData = LongestMatch(sourceData, dictionaryData); + + // If no match, or the match is too small, then take the next byte + // and emit as literal + if ((0 == candidateData.size)) // || (candidateData.size < 4)) + { + candidateData.size = 1; + candidateData.pData = sourceData.pData; + } + + // Adjust source stream + sourceData.pData += candidateData.size; + sourceData.size -= candidateData.size; + + dictionaryData.size = AddDictionary(candidateData, dictionaryData.size); + + if (candidateData.size > 3) + { + // Emit a dictionary reference + pDest += (int)EmitReference(pDest, (int)(candidateData.pData - dictionaryData.pData), candidateData); + bLastEmitIsLiteral = false; + } + else if (bLastEmitIsLiteral) + { + // Concatenate this literal onto the previous literal + pDest += ConcatLiteral(pLastLiteralDest, candidateData); + } + else + { + // Emit a new literal + pLastLiteralDest = pDest; + bLastEmitIsLiteral = true; + pDest += EmitLiteral(pDest, candidateData); + } + } + + return (int)(pDest - pOriginalDest); -//------------------------------------------------------------------------------ -// -// Decompress a Frame in the GSLA LZB Format -// -int LZBA_Decompress(unsigned char* pDest, unsigned char* pSource, unsigned char* pDataStart) -{ } //------------------------------------------------------------------------------ diff --git a/source/lzb.h b/source/lzb.h index d8d624e..964c985 100644 --- a/source/lzb.h +++ b/source/lzb.h @@ -14,7 +14,9 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize); // // LZB Compressor that uses GSLA Opcodes while encoding // -int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, unsigned char* pDataStart, unsigned char* pDictionary, int dictionarySize=0); +int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, + unsigned char* pDataStart, unsigned char* pDictionary, + int dictionarySize=0); int LZBA_Decompress(unsigned char* pDest, unsigned char* pSource, unsigned char* pDataStart); #endif // LZB_H From 0764a030e673b3bcf51a9e09c4978434e4be2cbc Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Sun, 19 Jul 2020 14:19:42 -0400 Subject: [PATCH 14/21] GSLA: encoder, work in progress --- source/gsla_file.cpp | 129 ++++++++++++++++++++++++-- source/gsla_file.h | 6 +- source/lzb.cpp | 214 +++++++++++++++++++++++++++++++++++++------ 3 files changed, 312 insertions(+), 37 deletions(-) diff --git a/source/gsla_file.cpp b/source/gsla_file.cpp index 3d9b8ce..ff441e3 100644 --- a/source/gsla_file.cpp +++ b/source/gsla_file.cpp @@ -242,7 +242,7 @@ void GSLAFile::LoadFromFile(const char* pFilePath) if (pINIT->IsValid()) { // We have an initial frame chunk - UnpackInitialFrame(pINIT); + UnpackInitialFrame(pINIT, pHeader); } else if (pANIM->IsValid()) { @@ -263,13 +263,13 @@ void GSLAFile::LoadFromFile(const char* pFilePath) // Unpack the initial frame, that's been packed with an empty initial dictionary // So every byte of the buffer will be written out (no skip opcodes) // -void GSLAFile::UnpackInitialFrame(GSLA_INIT* pINIT) +void GSLAFile::UnpackInitialFrame(GSLA_INIT* pINIT, GSLA_Header* pHeader) { unsigned char *pData = ((unsigned char*)pINIT) + sizeof(GSLA_INIT); unsigned char *pTargetBuffer = m_pC1PixelMaps[ 0 ]; // Data needs to be pre allocated - //DecompressFrame(pTargetBuffer, pData); + DecompressFrame(pTargetBuffer, pData, (unsigned char*)pHeader); } //------------------------------------------------------------------------------ @@ -359,9 +359,8 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) unsigned char* pInitialFrame = m_pC1PixelMaps[ 0 ]; // We're not worried about bank wrap on the first frame, and we don't have a pre-populated - // dictionary - int compressedSize = LZBA_Compress(pWorkBuffer, pInitialFrame, m_frameSize, - pInitialFrame, pInitialFrame); + // dictionary - Also use the best compression we can get here + int compressedSize = Old_LZB_Compress(pWorkBuffer, pInitialFrame, m_frameSize); for (int compressedIndex = 0; compressedIndex < compressedSize; ++compressedIndex) { @@ -458,4 +457,122 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) } //------------------------------------------------------------------------------ +// +// Std C memcpy seems to be stopping the copy from happening, when I overlap +// the buffer to get a pattern run copy (overlapped buffers) +// +static void my_memcpy(unsigned char* pDest, unsigned char* pSrc, int length) +{ + while (length-- > 0) + { + *pDest++ = *pSrc++; + } +} + +//------------------------------------------------------------------------------ +// +// pTarget is the Target Frame Buffer +// pData is the source data for a Frame +// +// pDataBaseAddress, is the base address wheret the animation file was loaded +// this is used so we can properly interpret bank-skip opcodes (data is broken +// into 64K chunks for the IIgs/65816) +// +// returns the number of bytes that have been processed in the pData +// +int GSLAFile::DecompressFrame(unsigned char* pTarget, unsigned char* pData, unsigned char* pDataBaseAddress) +{ + unsigned char *pDataStart = pData; + + int cursorPosition = 0; + unsigned short opcode; + + bool bDoWork = true; + + while (bDoWork) + { + opcode = pData[0]; + opcode |= (((unsigned short)pData[1])<<8); + + if (opcode & 0x8000) + { + if (opcode & 0x0001) + { + // Cursor Skip Forward + opcode = (opcode>>1) & 0x3FFF; + cursorPosition += (opcode+1); + pData+=2; + } + else + { + // Dictionary Copy + unsigned short dictionaryPosition = pData[2]; + dictionaryPosition |= (((unsigned short)pData[3])<<8); + + dictionaryPosition -= 0x2000; // it's like this to to help the + // GS decode it quicker + + unsigned short length = ((opcode>>1) & 0x3FFF)+1; + + my_memcpy(pTarget + cursorPosition, pTarget + dictionaryPosition, (int) length ); + + pData += 4; + cursorPosition += length; + } + } + else + { + if (opcode & 0x0001) + { + // Literal Copy Bytes + pData += 2; + unsigned short length = ((opcode>>1) & 0x3FFF)+1; + + my_memcpy(pTarget + cursorPosition, pData, (int) length); + + pData += length; + cursorPosition += length; + } + else + { + opcode = ((opcode>>1)) & 3; + + switch (opcode) + { + case 0: // Source bank Skip + { + int offset = (int)(pData - pDataBaseAddress); + + offset &= 0xFFFF0000; + offset += 0x00010000; + + pData = pDataBaseAddress + offset; + } + break; + case 1: // End of frame + pData+=2; + bDoWork = false; + break; + + case 2: // End of Animation + // Intentionally, leave cursor alone here + bDoWork = false; + break; + + default: + // Reserved / Illegal + bDoWork = false; + break; + } + + } + } + + + } + + return (int)(pData - pDataStart); +} + +//------------------------------------------------------------------------------ diff --git a/source/gsla_file.h b/source/gsla_file.h index 70bf6aa..493ea91 100644 --- a/source/gsla_file.h +++ b/source/gsla_file.h @@ -110,7 +110,7 @@ typedef struct GSLA_CHUNK class GSLAFile { public: - // Load in a C2 File + // Load in a GSLA File GSLAFile(const char *pFilePath); ~GSLAFile(); @@ -127,9 +127,11 @@ public: const std::vector& GetPixelMaps() { return m_pC1PixelMaps; } + int DecompressFrame(unsigned char* pTarget, unsigned char* pData, unsigned char* pDataBaseAddress); + private: - void UnpackInitialFrame(GSLA_INIT* pINIT); + void UnpackInitialFrame(GSLA_INIT* pINIT, GSLA_Header* pHeader); void UnpackAnimation(GSLA_ANIM* pANIM, GSLA_Header* pHeader); diff --git a/source/lzb.cpp b/source/lzb.cpp index 98caa18..03afada 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -29,13 +29,14 @@ static int DictionaryMatch(const DataString& data, int dictionarySize); // Stuff I need for a faster version static DataString LongestMatch(const DataString& data, const DataString& dictionary); +static DataString LongestMatch(const DataString& data, const DataString& dictionary, int cursorPosition); // // New Version, still Brute Force, but not as many times // int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) { - printf("LZB Compress %d bytes\n", sourceSize); + //printf("LZB Compress %d bytes\n", sourceSize); unsigned char *pOriginalDest = pDest; @@ -103,7 +104,7 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) // int Old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) { - printf("LZB_Compress %d bytes\n", sourceSize); + //printf("LZB_Compress %d bytes\n", sourceSize); // Initialize Dictionary int bytesInDictionary = 0; // eventually add the ability to start with the dictionary filled @@ -345,7 +346,102 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary) return result; } +//------------------------------------------------------------------------------ +DataString LongestMatch(const DataString& data, const DataString& dictionary, int cursorPosition) +{ + DataString result; + result.pData = nullptr; + result.size = 0; + // Find the longest matching data in the dictionary + if ((dictionary.size > 0) && (data.size > 0)) + { + DataString candidate; + candidate.pData = data.pData; + candidate.size = 0; + + // First Check for a pattern / run-length style match + // Check the end of the dictionary, to see if this data could be a + // pattern "run" (where we can repeat a pattern for X many times for free + // using the memcpy with overlapping source/dest buffers) + // (This is a dictionary based pattern run/length) + { + // Check for pattern sizes, start small + int max_pattern_size = 4096; + if (dictionary.size < max_pattern_size) max_pattern_size = dictionary.size; + if (data.size < max_pattern_size) max_pattern_size = data.size; + + for (int pattern_size = 1; pattern_size <= max_pattern_size; ++pattern_size) + { + int pattern_start = dictionary.size - pattern_size; + + for (int dataIndex = 0; dataIndex < data.size; ++dataIndex) + { + if (data.pData[ dataIndex ] == dictionary.pData[ pattern_start + (dataIndex % pattern_size) ]) + { + candidate.pData = dictionary.pData + pattern_start; + candidate.size = dataIndex+1; + continue; + } + + break; + } + + //if (candidate.size < pattern_size) + // break; + + if (candidate.size > result.size) + { + result = candidate; + } + } + } + + // As an optimization + int dictionarySize = dictionary.size; // - 1; // This last string has already been checked by, the + // run-length matcher above + + // As the size grows, we're missing potential matches in here + // I think the best way to counter this is to attempt somthing + // like KMP + + if (dictionarySize > candidate.size) + { + // Check the dictionary for a match, brute force + for (int dictionaryIndex = 0; dictionaryIndex <= (dictionarySize-candidate.size); ++dictionaryIndex) + { + int sizeAvailable = dictionarySize - dictionaryIndex; + + if (sizeAvailable > data.size) sizeAvailable = data.size; + + // this could index off the end of the dictionary!!! FIX ME + for (int dataIndex = 0; dataIndex < sizeAvailable; ++dataIndex) + { + if (data.pData[ dataIndex ] == dictionary.pData[ dictionaryIndex + dataIndex ]) + { + if (dataIndex >= candidate.size) + { + candidate.pData = dictionary.pData + dictionaryIndex; + candidate.size = dataIndex + 1; + } + continue; + } + + break; + } + + if (candidate.size > result.size) + { + result = candidate; + //dictionaryIndex = -1; + break; + } + } + } + } + + return result; +} //------------------------------------------------------------------------------ // @@ -576,6 +672,11 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) //------------------------------------------------------------------------------ // // Compress a Frame in the GSLA LZB Format +// +// The dictionary is also the canvas, so when we're finished the dictionary +// buffer will match the original pSource buffer +// +// If they both match to begin with, we just crap out an End of Frame opcode // int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, unsigned char* pDataStart, unsigned char* pDictionary, @@ -583,6 +684,10 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, { // printf("LZBA Compress %d bytes\n", sourceSize); + // Used for bank skip opcode emission + int bankOffset = (int)((pDest - pDataStart) & 0xFFFF); + + // So we can track how big our compressed data ends up being unsigned char *pOriginalDest = pDest; DataString sourceData; @@ -601,44 +706,95 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, bool bLastEmitIsLiteral = false; unsigned char* pLastLiteralDest = nullptr; - while (sourceData.size > 0) + int lastEmittedCursorPosition = 0; // This is the default for each frame + + for (int cursorPosition = 0; cursorPosition < dictionarySize;) { - candidateData = LongestMatch(sourceData, dictionaryData); - - // If no match, or the match is too small, then take the next byte - // and emit as literal - if ((0 == candidateData.size)) // || (candidateData.size < 4)) + if (pSource[ cursorPosition ] != pDictionary[ cursorPosition ]) { - candidateData.size = 1; - candidateData.pData = sourceData.pData; - } + // Here is some data that has to be processed, so let's decide + // how large of a chunk of data we're looking at here - // Adjust source stream - sourceData.pData += candidateData.size; - sourceData.size -= candidateData.size; + // Do we need to emit a Skip opcode?, compare cursor to last emit + // and emit a skip command if we need it (I'm going want a gap of + // at least 3 bytes? before we call it the end + int tempCursorPosition = cursorPosition; + int gapCount = 0; + for (; tempCursorPosition < dictionarySize; ++tempCursorPosition) + { + if (pSource[ cursorPosition ] != pDictionary[ cursorPosition ]) + { + gapCount = 0; + } + else + { + gapCount++; + if (gapCount >= 3) + break; + } + } - dictionaryData.size = AddDictionary(candidateData, dictionaryData.size); + tempCursorPosition -= gapCount; - if (candidateData.size > 3) - { - // Emit a dictionary reference - pDest += (int)EmitReference(pDest, (int)(candidateData.pData - dictionaryData.pData), candidateData); - bLastEmitIsLiteral = false; - } - else if (bLastEmitIsLiteral) - { - // Concatenate this literal onto the previous literal - pDest += ConcatLiteral(pLastLiteralDest, candidateData); + // Now we know from cursorPosition to tempCursorPosition is data + // that we want to encode, we either literally copy it, or look + // to see if this data is already in the dictionary (so we can copy + // it from one part of the frame buffer to another part) + + sourceData.pData = &pSource[ cursorPosition ]; + sourceData.size = tempCursorPosition - cursorPosition; + + while (sourceData.size > 0) + { + candidateData = LongestMatch(sourceData, dictionaryData, cursorPosition); + + // If no match, or the match is too small, then take the next byte + // and emit as literal + if ((0 == candidateData.size)) // || (candidateData.size < 4)) + { + candidateData.size = 1; + candidateData.pData = sourceData.pData; + } + + // Adjust source stream + sourceData.pData += candidateData.size; + sourceData.size -= candidateData.size; + + // Modify the dictionary + cursorPosition = AddDictionary(candidateData, cursorPosition); + lastEmittedCursorPosition = cursorPosition; + + if (candidateData.size > 3) + { + // Emit a dictionary reference + pDest += (int)EmitReference(pDest, (int)(candidateData.pData - dictionaryData.pData), candidateData); + bLastEmitIsLiteral = false; + } + else if (bLastEmitIsLiteral) + { + // Concatenate this literal onto the previous literal + pDest += ConcatLiteral(pLastLiteralDest, candidateData); + } + else + { + // Emit a new literal + pLastLiteralDest = pDest; + bLastEmitIsLiteral = true; + pDest += EmitLiteral(pDest, candidateData); + } + } } else { - // Emit a new literal - pLastLiteralDest = pDest; - bLastEmitIsLiteral = true; - pDest += EmitLiteral(pDest, candidateData); + // no change + cursorPosition++; } } + // Emit the End of Frame Opcode + *pDest++ = 0x02; + *pDest++ = 0x00; + return (int)(pDest - pOriginalDest); } From 6956281d1e2668a32abf6ae33f78800107c8bcb0 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Sun, 19 Jul 2020 15:05:17 -0400 Subject: [PATCH 15/21] GSLA Encoder - WIP - I think all that's left to do is to emit source bank skip opcodes, so the player can deal deal with data > 64K in size --- source/lzb.cpp | 123 ++++++++++++++++++++++-------------------------- source/lzb.h | 4 +- source/main.cpp | 56 ---------------------- 3 files changed, 58 insertions(+), 125 deletions(-) diff --git a/source/lzb.cpp b/source/lzb.cpp index 03afada..13371f8 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -8,8 +8,27 @@ #include "bctypes.h" +// +// This is written specifically for the GSLA, so opcodes emitted are designed +// to work with our version of a run/skip/dump +// + +// +//Command Word, encoded low-high, what the bits mean: +// +// xxx_xxxx_xxxx_xxx is the number of bytes 1-16384 to follow (0 == 1 byte) +// +//%0xxx_xxxx_xxxx_xxx1 - Copy Bytes - straight copy bytes +//%1xxx_xxxx_xxxx_xxx1 - Skip Bytes - skip bytes / move the cursor +//%1xxx_xxxx_xxxx_xxx0 - Dictionary Copy Bytes from frame buffer to frame buffer +// +//%0000_0000_0000_0000- Source Skip -> Source pointer skips to next bank of data +//%0000_0000_0000_0010- End of Frame - end of frame +//%0000_0000_0000_0110- End of Animation / End of File / no more frames +// + #define MAX_DICTIONARY_SIZE (32 * 1024) -#define MAX_STRING_SIZE (16383) +#define MAX_STRING_SIZE (16384) // // Yes This is a 32K Buffer, of bytes, with no structure to it // @@ -235,11 +254,11 @@ int Old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSiz // Return new dictionarySize static int AddDictionary(const DataString& data, int dictionarySize) { - //int dataIndex = 0; - //while ((dictionarySize < MAX_DICTIONARY_SIZE) && (dataIndex < data.size)) - //{ - // pDictionary[ dictionarySize++ ] = data.pData[ dataIndex++ ]; - //} + int dataIndex = 0; + while (dataIndex < data.size) + { + pDictionary[ dictionarySize++ ] = data.pData[ dataIndex++ ]; + } dictionarySize += data.size; @@ -537,10 +556,18 @@ static int ConcatLiteral(unsigned char *pDest, DataString& data) int opCode = pDest[0]; opCode |= (int)(((pDest[1])&0x7F)<<8); + opCode>>=1; + opCode+=1; + // opCode contains the length of the literal that's already encoded + int skip = opCode; opCode += outSize; // Opcode + opCode -= 1; + opCode <<=1; + opCode |= 1; + *pDest++ = (unsigned char)(opCode & 0xFF); *pDest++ = (unsigned char)((opCode >> 8) & 0x7F); @@ -566,9 +593,15 @@ static int EmitLiteral(unsigned char *pDest, DataString& data) // Return Size int outSize = 2 + (int)data.size; - // Opcode - *pDest++ = (unsigned char)(data.size & 0xFF); - *pDest++ = (unsigned char)((data.size >> 8) & 0x7F); + unsigned short length = (unsigned short)data.size; + length -= 1; + + unsigned short opcode = length<<1; + opcode |= 0x0001; + + // Opcode out + *pDest++ = (unsigned char)( opcode & 0xFF ); + *pDest++ = (unsigned char)(( opcode>>8)&0xFF); // Literal Data for (int idx = 0; idx < data.size; ++idx) @@ -590,12 +623,22 @@ static int EmitReference(unsigned char *pDest, int dictionaryOffset, DataString& // Return Size int outSize = 2 + 2; - // Opcode - *pDest++ = (unsigned char)(data.size & 0xFF); - *pDest++ = (unsigned char)((data.size >> 8) & 0x7F) | 0x80; + unsigned short length = (unsigned short)data.size; + length -= 1; - *pDest++ = (unsigned char)(dictionaryOffset & 0xFF); - *pDest++ = (unsigned char)((dictionaryOffset>>8) & 0xFF); + unsigned short opcode = length<<1; + opcode |= 0x8000; + + // Opcode out + *pDest++ = (unsigned char)( opcode & 0xFF ); + *pDest++ = (unsigned char)(( opcode>>8)&0xFF); + + // Destination Address out + unsigned short address = (unsigned short)dictionaryOffset; + address += 0x2000; // So we don't have to add $2000 in the animation player + + *pDest++ = (unsigned char)(address & 0xFF); + *pDest++ = (unsigned char)((address>>8)&0xFF); // Clear data.pData += data.size; @@ -617,58 +660,6 @@ static void my_memcpy(u8* pDest, u8* pSrc, int length) } } -//------------------------------------------------------------------------------ -// -// Simple Decompress, for validation -// -void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize) -{ - int decompressedBytes = 0; - - unsigned char *pOriginalSource = pSource; - - while (decompressedBytes < destSize) - { - u16 opcode = *pSource++; - opcode |= ((u16)(*pSource++))<<8; - -// printf("%04X:", (unsigned int)(pSource-pOriginalSource)); - - - if (opcode & 0x8000) - { - // Dictionary - opcode &= 0x7FFF; - - // Dictionary Copy from the output stream - u16 offset = *pSource++; - offset |= ((u16)(*pSource++))<<8; - - const char* overlapped = ""; - - if ((&pDest[ decompressedBytes ] - &pDest[ offset ]) < opcode) - { - overlapped = "pattern"; - } - - my_memcpy(&pDest[ decompressedBytes ], &pDest[ offset ], opcode); - decompressedBytes += opcode; - - -// printf("%04X:Dic %04X %s\n",decompressedBytes, (unsigned int)opcode, overlapped); - } - else - { - // Literal Copy, from compressed stream - memcpy(&pDest[ decompressedBytes ], pSource, opcode); - decompressedBytes += opcode; - pSource += opcode; - -// printf("%04X:Lit %04X\n",decompressedBytes, (unsigned int)opcode); - } - } -} - //------------------------------------------------------------------------------ // // Compress a Frame in the GSLA LZB Format diff --git a/source/lzb.h b/source/lzb.h index 964c985..7d67c09 100644 --- a/source/lzb.h +++ b/source/lzb.h @@ -1,5 +1,5 @@ // -// LZB Encode / Decode +// LZB Encode // #ifndef LZB_H #define LZB_H @@ -9,7 +9,6 @@ // int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize); int Old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize); -void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize); // // LZB Compressor that uses GSLA Opcodes while encoding @@ -17,7 +16,6 @@ void LZB_Decompress(unsigned char* pDest, unsigned char* pSource, int destSize); int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, unsigned char* pDataStart, unsigned char* pDictionary, int dictionarySize=0); -int LZBA_Decompress(unsigned char* pDest, unsigned char* pSource, unsigned char* pDataStart); #endif // LZB_H diff --git a/source/main.cpp b/source/main.cpp index 699dada..907be1f 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -11,8 +11,6 @@ #include "c2_file.h" #include "gsla_file.h" -#include "lzb.h" - //------------------------------------------------------------------------------ static void helpText() { @@ -130,60 +128,6 @@ int main(int argc, char* argv[]) anim.SaveToFile(pOutfilePath); } - - #if 0 - const std::vector& c1Datas = c2data.GetPixelMaps(); - - // LZB Testing Code, that I should probalby remove from this file - // Just ignore this block of code - #if 1 - unsigned char workbuffer[64*1024]; - for (int idx = 0; idx < frameCount; ++idx) - { - int compressedSize = LZB_Compress(workbuffer, c1Datas[ idx ], 32 * 1024); - printf("compressedSize = %d\n", compressedSize); - unsigned char validationBuffer[ 32 * 1024 ]; - LZB_Decompress(validationBuffer, workbuffer, 32 * 1024); - if (0 == memcmp(c1Datas[ idx ], validationBuffer, 32*1024)) - { - printf("Decompression Validated\n"); - } - else - { - printf("Decompression Corrupted\n"); - } - } - - #else - unsigned char workbuffer[64*1024]; - unsigned char workbuffer2[64*1024]; - - for (int idx = 0; idx < frameCount; ++idx) - { - int oldCompressedSize = Old_LZB_Compress(workbuffer2, c1Datas[ idx ], 32 * 1024); - printf("old compressedSize = %d\n", oldCompressedSize); - int compressedSize = LZB_Compress(workbuffer, c1Datas[ idx ], 32 * 1024); - printf("compressedSize = %d\n", compressedSize); - - unsigned char validationBuffer[ 32 * 1024 ]; - - printf("Decompress OLD\n"); - LZB_Decompress(validationBuffer, workbuffer2, 32 * 1024); - printf("Decompress NEW\n"); - LZB_Decompress(validationBuffer, workbuffer, 32 * 1024); - - if (0 == memcmp(c1Datas[ idx ], validationBuffer, 32*1024)) - { - printf("Decompression Validated\n"); - } - else - { - printf("Decompression Corrupted\n"); - } - - } - #endif - #endif } } From 8dfb3028c72d6f648b6a179e45b7afe69f4fb96b Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Sun, 19 Jul 2020 16:08:09 -0400 Subject: [PATCH 16/21] GSLA: might have a version 1 encoder finished... all that's left is to debug / test / make sure the thing works --- source/gsla_file.cpp | 6 +- source/lzb.cpp | 160 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 154 insertions(+), 12 deletions(-) diff --git a/source/gsla_file.cpp b/source/gsla_file.cpp index ff441e3..6657da5 100644 --- a/source/gsla_file.cpp +++ b/source/gsla_file.cpp @@ -403,7 +403,7 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) { // I don't want random data in the bank gaps, so initialize this // buffer with zero - memset(pWorkBuffer, 0, m_frameSize * 2); + //memset(pWorkBuffer, 0, m_frameSize * 2); int frameSize = LZBA_Compress(pWorkBuffer, m_pC1PixelMaps[ frameIndex], m_frameSize, pWorkBuffer-bytes.size(), @@ -416,7 +416,7 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) } // Add the RING Frame - memset(pWorkBuffer, 0, m_frameSize * 2); + //memset(pWorkBuffer, 0, m_frameSize * 2); int ringSize = LZBA_Compress(pWorkBuffer, m_pC1PixelMaps[ 0 ], m_frameSize, pWorkBuffer-bytes.size(), @@ -430,6 +430,8 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) delete[] pCanvas; pCanvas = nullptr; // Insert End of file/ End of Animation Done opcode + // -- There has to be room for this, or there wouldn't be room to insert + // -- a source bank skip opcode bytes.push_back( 0x06 ); bytes.push_back( 0x00 ); diff --git a/source/lzb.cpp b/source/lzb.cpp index 13371f8..1f34859 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -8,6 +8,8 @@ #include "bctypes.h" +#include "assert.h" + // // This is written specifically for the GSLA, so opcodes emitted are designed // to work with our version of a run/skip/dump @@ -387,12 +389,12 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary, in { // Check for pattern sizes, start small int max_pattern_size = 4096; - if (dictionary.size < max_pattern_size) max_pattern_size = dictionary.size; + if (cursorPosition < max_pattern_size) max_pattern_size = cursorPosition; if (data.size < max_pattern_size) max_pattern_size = data.size; for (int pattern_size = 1; pattern_size <= max_pattern_size; ++pattern_size) { - int pattern_start = dictionary.size - pattern_size; + int pattern_start = cursorPosition - pattern_size; for (int dataIndex = 0; dataIndex < data.size; ++dataIndex) { @@ -406,9 +408,6 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary, in break; } - //if (candidate.size < pattern_size) - // break; - if (candidate.size > result.size) { result = candidate; @@ -416,9 +415,9 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary, in } } - // As an optimization - int dictionarySize = dictionary.size; // - 1; // This last string has already been checked by, the - // run-length matcher above + // This will keep us from finding matches that we can't use + + int dictionarySize = cursorPosition; // As the size grows, we're missing potential matches in here // I think the best way to counter this is to attempt somthing @@ -457,6 +456,43 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary, in } } } + + // Look for matches beyond the cursor + dictionarySize = dictionary.size; + + if ((dictionarySize-cursorPosition) > candidate.size) + { + // Check the dictionary for a match, brute force + for (int dictionaryIndex = cursorPosition; dictionaryIndex <= (dictionarySize-candidate.size); ++dictionaryIndex) + { + int sizeAvailable = dictionarySize - dictionaryIndex; + + if (sizeAvailable > data.size) sizeAvailable = data.size; + + // this could index off the end of the dictionary!!! FIX ME + for (int dataIndex = 0; dataIndex < sizeAvailable; ++dataIndex) + { + if (data.pData[ dataIndex ] == dictionary.pData[ dictionaryIndex + dataIndex ]) + { + if (dataIndex >= candidate.size) + { + candidate.pData = dictionary.pData + dictionaryIndex; + candidate.size = dataIndex + 1; + } + continue; + } + + break; + } + + if (candidate.size > result.size) + { + result = candidate; + break; + } + } + } + } return result; @@ -660,6 +696,63 @@ static void my_memcpy(u8* pDest, u8* pSrc, int length) } } +//------------------------------------------------------------------------------ +// +// Emit a Cursor Skip forward opcode +// +int EmitSkip(unsigned char* pDest, int skipSize) +{ + int outSize = 2; + + unsigned short length = (unsigned short)skipSize; + length -= 1; + + unsigned short opcode = length<<1; + opcode |= 0x8001; + // Opcode out + *pDest++ = (unsigned char)( opcode & 0xFF ); + *pDest++ = (unsigned char)(( opcode>>8)&0xFF); + + return outSize; +} + +//------------------------------------------------------------------------------ +// +// Forcibly Emit a source Skip Opcode +// +// return space_left_in_Bank +// +int EmitSourceSkip(unsigned char*& pDest, int space_left_in_bank) +{ + assert(space_left_in_bank >= 2); + + *pDest++ = 0; + *pDest++ = 0; + space_left_in_bank-=2; + + while (space_left_in_bank) + { + space_left_in_bank--; + *pDest++ = 0; + } + + return 0x10000; +} + +//------------------------------------------------------------------------------ +// +// Conditionally shit out the Source Bank Skip +// +int CheckEmitSourceSkip(int checkSpace, unsigned char*& pDest, int space_left_in_bank) +{ + if ((checkSpace+2) > space_left_in_bank) + { + return EmitSourceSkip(pDest, space_left_in_bank); + } + + return space_left_in_bank; +} + //------------------------------------------------------------------------------ // // Compress a Frame in the GSLA LZB Format @@ -699,6 +792,10 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, int lastEmittedCursorPosition = 0; // This is the default for each frame + int space_left_in_bank = (int)0x10000 - (int)((pDest - pDataStart)&0xFFFF); + + space_left_in_bank = CheckEmitSourceSkip(0, pDest, space_left_in_bank); + for (int cursorPosition = 0; cursorPosition < dictionarySize;) { if (pSource[ cursorPosition ] != pDictionary[ cursorPosition ]) @@ -709,6 +806,20 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, // Do we need to emit a Skip opcode?, compare cursor to last emit // and emit a skip command if we need it (I'm going want a gap of // at least 3 bytes? before we call it the end + int skipSize = cursorPosition - lastEmittedCursorPosition; + + if (skipSize) + { + space_left_in_bank = CheckEmitSourceSkip(2, pDest, space_left_in_bank); + + // We need to Skip + pDest += EmitSkip(pDest, skipSize); + bLastEmitIsLiteral = false; + lastEmittedCursorPosition = cursorPosition; + + space_left_in_bank = (int)0x10000 - (int)((pDest - pDataStart)&0xFFFF); + } + int tempCursorPosition = cursorPosition; int gapCount = 0; for (; tempCursorPosition < dictionarySize; ++tempCursorPosition) @@ -757,21 +868,48 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, if (candidateData.size > 3) { + space_left_in_bank = CheckEmitSourceSkip(4, pDest, space_left_in_bank); + // Emit a dictionary reference pDest += (int)EmitReference(pDest, (int)(candidateData.pData - dictionaryData.pData), candidateData); bLastEmitIsLiteral = false; + + space_left_in_bank = (int)0x10000 - (int)((pDest - pDataStart)&0xFFFF); } else if (bLastEmitIsLiteral) { - // Concatenate this literal onto the previous literal - pDest += ConcatLiteral(pLastLiteralDest, candidateData); + // This is a problem for the source bank skip, we can't + // concatenate if we end up injecting a source bank skip opcode + // into the stream... what to do???, if insert, we will need to + // do a "normal" literal emission, ugly + + int space = CheckEmitSourceSkip(candidateData.size, pDest, space_left_in_bank); + + if (space != space_left_in_bank) + { + // Emit a new literal + pLastLiteralDest = pDest; + bLastEmitIsLiteral = true; + pDest += EmitLiteral(pDest, candidateData); + + space_left_in_bank = (int)0x10000 - (int)((pDest - pDataStart)&0xFFFF); + } + else + { + // Concatenate this literal onto the previous literal + pDest += ConcatLiteral(pLastLiteralDest, candidateData); + } } else { + space_left_in_bank = CheckEmitSourceSkip(2 + candidateData.size, pDest, space_left_in_bank); + // Emit a new literal pLastLiteralDest = pDest; bLastEmitIsLiteral = true; pDest += EmitLiteral(pDest, candidateData); + + space_left_in_bank = (int)0x10000 - (int)((pDest - pDataStart)&0xFFFF); } } } @@ -782,6 +920,8 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, } } + space_left_in_bank = CheckEmitSourceSkip(2, pDest, space_left_in_bank); + // Emit the End of Frame Opcode *pDest++ = 0x02; *pDest++ = 0x00; From 1c0f4d5ae39eee78091d3dbe334d6b482045d6e6 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Sun, 19 Jul 2020 17:21:04 -0400 Subject: [PATCH 17/21] GSLA: seems to be spitting out a file, but is is valid? --- source/gsla_file.cpp | 15 +++++++++++++-- source/lzb.cpp | 8 +++++--- source/main.cpp | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/source/gsla_file.cpp b/source/gsla_file.cpp index 6657da5..1645753 100644 --- a/source/gsla_file.cpp +++ b/source/gsla_file.cpp @@ -351,6 +351,8 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) pINIT->I = 'I'; pINIT->N = 'N'; pINIT->i = 'I'; pINIT->T = 'T'; pINIT->chunk_length = 0; // temp chunk size + printf("Save Initial Frame\n"); + // Need a place to put compressed data, in theory it could be bigger // than the original data, I think if that happens, the image was probably // designed to break this, anyway, give double theoretical max @@ -362,6 +364,8 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) // dictionary - Also use the best compression we can get here int compressedSize = Old_LZB_Compress(pWorkBuffer, pInitialFrame, m_frameSize); + printf("frameSize = %d\n", compressedSize); + for (int compressedIndex = 0; compressedIndex < compressedSize; ++compressedIndex) { bytes.push_back(pWorkBuffer[ compressedIndex ]); @@ -396,11 +400,11 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) unsigned char *pCanvas = new unsigned char[ m_frameSize ]; memcpy(pCanvas, m_pC1PixelMaps[0], m_frameSize); - memcpy(pCanvas, m_pC1PixelMaps[0], m_frameSize); - // Let's encode some frames buddy for (int frameIndex = 1; frameIndex < m_pC1PixelMaps.size(); ++frameIndex) { + printf("Save Frame %d\n", frameIndex+1); + // I don't want random data in the bank gaps, so initialize this // buffer with zero //memset(pWorkBuffer, 0, m_frameSize * 2); @@ -409,6 +413,9 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) m_frameSize, pWorkBuffer-bytes.size(), pCanvas, m_frameSize ); + printf("frameSize = %d\n", frameSize); + + for (int frameIndex = 0; frameIndex < frameSize; ++frameIndex) { bytes.push_back(pWorkBuffer[ frameIndex ]); @@ -418,10 +425,14 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) // Add the RING Frame //memset(pWorkBuffer, 0, m_frameSize * 2); + printf("Save Ring Frame\n"); + int ringSize = LZBA_Compress(pWorkBuffer, m_pC1PixelMaps[ 0 ], m_frameSize, pWorkBuffer-bytes.size(), pCanvas, m_frameSize ); + printf("Ring Size %d\n", ringSize); + for (int ringIndex = 0; ringIndex < ringSize; ++ringIndex) { bytes.push_back(pWorkBuffer[ ringIndex ]); diff --git a/source/lzb.cpp b/source/lzb.cpp index 1f34859..1a0b06d 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -262,7 +262,7 @@ static int AddDictionary(const DataString& data, int dictionarySize) pDictionary[ dictionarySize++ ] = data.pData[ dataIndex++ ]; } - dictionarySize += data.size; + //dictionarySize += data.size; return dictionarySize; } @@ -824,7 +824,7 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, int gapCount = 0; for (; tempCursorPosition < dictionarySize; ++tempCursorPosition) { - if (pSource[ cursorPosition ] != pDictionary[ cursorPosition ]) + if (pSource[ tempCursorPosition ] != pDictionary[ tempCursorPosition ]) { gapCount = 0; } @@ -844,7 +844,9 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, // it from one part of the frame buffer to another part) sourceData.pData = &pSource[ cursorPosition ]; - sourceData.size = tempCursorPosition - cursorPosition; + sourceData.size = tempCursorPosition - cursorPosition + 1; + + cursorPosition = tempCursorPosition; while (sourceData.size > 0) { diff --git a/source/main.cpp b/source/main.cpp index 907be1f..4df35bc 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -124,7 +124,7 @@ int main(int argc, char* argv[]) GSLAFile anim(320,200, 0x8000); anim.AddImages(c1Datas); - + anim.SaveToFile(pOutfilePath); } From c768f341568c804352ba38f2f32e614156e3479a Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Sun, 19 Jul 2020 17:58:59 -0400 Subject: [PATCH 18/21] Add animation decode verification code, to make sure things seems to work (hint, they do not work yet) --- source/gsla_file.cpp | 24 +++++++++++++++++------- source/gsla_file.h | 1 + source/main.cpp | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/source/gsla_file.cpp b/source/gsla_file.cpp index 1645753..1add8b1 100644 --- a/source/gsla_file.cpp +++ b/source/gsla_file.cpp @@ -265,9 +265,9 @@ void GSLAFile::LoadFromFile(const char* pFilePath) // void GSLAFile::UnpackInitialFrame(GSLA_INIT* pINIT, GSLA_Header* pHeader) { - unsigned char *pData = ((unsigned char*)pINIT) + sizeof(GSLA_INIT); + unsigned char* pData = ((unsigned char*)pINIT) + sizeof(GSLA_INIT); - unsigned char *pTargetBuffer = m_pC1PixelMaps[ 0 ]; // Data needs to be pre allocated + unsigned char* pTargetBuffer = m_pC1PixelMaps[ 0 ]; // Data needs to be pre allocated DecompressFrame(pTargetBuffer, pData, (unsigned char*)pHeader); } @@ -278,11 +278,21 @@ void GSLAFile::UnpackInitialFrame(GSLA_INIT* pINIT, GSLA_Header* pHeader) // void GSLAFile::UnpackAnimation(GSLA_ANIM* pANIM, GSLA_Header* pHeader) { - unsigned char *pData = ((unsigned char*)pANIM) + sizeof(GSLA_ANIM); + unsigned char* pData = ((unsigned char*)pANIM) + sizeof(GSLA_ANIM); - unsigned char *pTargetBuffer = m_pC1PixelMaps[ 0 ]; // Data needs to be pre allocated + unsigned char *pCanvas = new unsigned char[m_frameSize]; - //DecompressAnim(pTargetBuffer, pData); + // Initialize the Canvas with the first frame + memcpy(pCanvas, m_pC1PixelMaps[0], m_frameSize); + + for (int idx = 1; idx < m_pC1PixelMaps.size(); ++idx) + { + // Apply Changes to the Canvas + pData += DecompressFrame(pCanvas, pData, (unsigned char*) pHeader); + + // Capture the Canvas + memcpy(m_pC1PixelMaps[idx], pCanvas, m_frameSize); + } } //------------------------------------------------------------------------------ @@ -318,7 +328,7 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) // Add the header bytes.resize( bytes.size() + sizeof(GSLA_Header) ); - //$$JGA Rememeber, you have to set the pointer, before every access + //$$JGA Remember, you have to set the pointer, before every access //$$JGA to the header data, because vector is going to change out //$$JGA memory addresses from underneath you GSLA_Header* pHeader = (GSLA_Header*)&bytes[0]; @@ -567,7 +577,7 @@ int GSLAFile::DecompressFrame(unsigned char* pTarget, unsigned char* pData, unsi bDoWork = false; break; - case 2: // End of Animation + case 3: // End of Animation // Intentionally, leave cursor alone here bDoWork = false; break; diff --git a/source/gsla_file.h b/source/gsla_file.h index 493ea91..d51e345 100644 --- a/source/gsla_file.h +++ b/source/gsla_file.h @@ -124,6 +124,7 @@ public: int GetFrameCount() { return (int)m_pC1PixelMaps.size(); } int GetWidth() { return m_widthPixels; } int GetHeight() { return m_heightPixels; } + int GetFrameSize() { return m_frameSize; } const std::vector& GetPixelMaps() { return m_pC1PixelMaps; } diff --git a/source/main.cpp b/source/main.cpp index 4df35bc..e780553 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -127,6 +127,21 @@ int main(int argc, char* argv[]) anim.SaveToFile(pOutfilePath); + #if 1 + { + // Verify the conversion is good + // Load the file back in + GSLAFile verify(pOutfilePath); + + const std::vector &frames = verify.GetPixelMaps(); + + for (int idx = 0; idx < frames.size(); ++idx) + { + int result = memcmp(c1Datas[idx % c1Datas.size()], frames[idx], verify.GetFrameSize()); + printf("Verify Frame %d - %s\n", idx, result ? "Failed" : "Good"); + } + } + #endif } } From 8dd7fbd28fd8863a3d6dd335638fd89149aad8c8 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Mon, 20 Jul 2020 08:58:20 -0400 Subject: [PATCH 19/21] gsla: all the frames in my test file are decompressing correctly --- source/gsla_file.cpp | 7 +++- source/lzb.cpp | 85 ++++++++++++++++++++++++++++++++------------ source/lzb.h | 2 +- 3 files changed, 70 insertions(+), 24 deletions(-) diff --git a/source/gsla_file.cpp b/source/gsla_file.cpp index 1add8b1..24b2a21 100644 --- a/source/gsla_file.cpp +++ b/source/gsla_file.cpp @@ -419,10 +419,15 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) // buffer with zero //memset(pWorkBuffer, 0, m_frameSize * 2); - int frameSize = LZBA_Compress(pWorkBuffer, m_pC1PixelMaps[ frameIndex], + int frameSize = LZBA_Compress(pWorkBuffer, m_pC1PixelMaps[ frameIndex ], m_frameSize, pWorkBuffer-bytes.size(), pCanvas, m_frameSize ); + //int canvasDiff = memcmp(pCanvas, m_pC1PixelMaps[ frameIndex], m_frameSize); + //if (canvasDiff) + //{ + // printf("Canvas is not correct - %d\n", canvasDiff); + //} printf("frameSize = %d\n", frameSize); diff --git a/source/lzb.cpp b/source/lzb.cpp index 1a0b06d..7af629c 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -34,7 +34,7 @@ // // Yes This is a 32K Buffer, of bytes, with no structure to it // -static unsigned char *pDictionary = nullptr; +static unsigned char *pGlobalDictionary = nullptr; struct DataString { // Information about the data we're trying to match @@ -70,6 +70,7 @@ int LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize) sourceData.size = sourceSize; // Remember, this eventually will point at the frame buffer + pGlobalDictionary = pSource; dictionaryData.pData = pSource; dictionaryData.size = 0; @@ -129,7 +130,7 @@ int Old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSiz // Initialize Dictionary int bytesInDictionary = 0; // eventually add the ability to start with the dictionary filled - pDictionary = pSource; + pGlobalDictionary = pSource; int processedBytes = 0; int bytesEmitted = 0; @@ -259,7 +260,7 @@ static int AddDictionary(const DataString& data, int dictionarySize) int dataIndex = 0; while (dataIndex < data.size) { - pDictionary[ dictionarySize++ ] = data.pData[ dataIndex++ ]; + pGlobalDictionary[ dictionarySize++ ] = data.pData[ dataIndex++ ]; } //dictionarySize += data.size; @@ -508,7 +509,7 @@ static int DictionaryMatch(const DataString& data, int dictionarySize) { if( (0 == dictionarySize ) || (0 == data.size) || - (data.size > 16384) ) // 16384 is largest string copy we can encode + (data.size > MAX_STRING_SIZE) ) // 16384 is largest string copy we can encode { return -1; } @@ -531,7 +532,7 @@ static int DictionaryMatch(const DataString& data, int dictionarySize) for (int dataIndex = 0; dataIndex < data.size; ++dataIndex) { - if (data.pData[ dataIndex ] == pDictionary[ pattern_start + (dataIndex % pattern_size) ]) + if (data.pData[ dataIndex ] == pGlobalDictionary[ pattern_start + (dataIndex % pattern_size) ]) continue; bMatch = false; @@ -563,7 +564,7 @@ static int DictionaryMatch(const DataString& data, int dictionarySize) bool bMatch = true; for (int dataIdx = 0; dataIdx < data.size; ++dataIdx) { - if (data.pData[ dataIdx ] == pDictionary[ idx + dataIdx ]) + if (data.pData[ dataIdx ] == pGlobalDictionary[ idx + dataIdx ]) continue; bMatch = false; @@ -632,6 +633,8 @@ static int EmitLiteral(unsigned char *pDest, DataString& data) unsigned short length = (unsigned short)data.size; length -= 1; + assert(length < MAX_STRING_SIZE); + unsigned short opcode = length<<1; opcode |= 0x0001; @@ -662,6 +665,8 @@ static int EmitReference(unsigned char *pDest, int dictionaryOffset, DataString& unsigned short length = (unsigned short)data.size; length -= 1; + assert(length < MAX_STRING_SIZE); + unsigned short opcode = length<<1; opcode |= 0x8000; @@ -698,20 +703,36 @@ static void my_memcpy(u8* pDest, u8* pSrc, int length) //------------------------------------------------------------------------------ // -// Emit a Cursor Skip forward opcode +// Emit one or more Cursor Skip forward opcode // int EmitSkip(unsigned char* pDest, int skipSize) { - int outSize = 2; + int outSize = 0; + int thisSkip = 0; - unsigned short length = (unsigned short)skipSize; - length -= 1; + while (skipSize > 0) + { + outSize+=2; - unsigned short opcode = length<<1; - opcode |= 0x8001; - // Opcode out - *pDest++ = (unsigned char)( opcode & 0xFF ); - *pDest++ = (unsigned char)(( opcode>>8)&0xFF); + thisSkip = skipSize; + if (thisSkip > MAX_STRING_SIZE) + { + thisSkip = MAX_STRING_SIZE; + } + skipSize -= thisSkip; + + + unsigned short length = (unsigned short)thisSkip; + length -= 1; + + assert(length < MAX_STRING_SIZE); + + unsigned short opcode = length<<1; + opcode |= 0x8001; + // Opcode out + *pDest++ = (unsigned char)( opcode & 0xFF ); + *pDest++ = (unsigned char)(( opcode>>8)&0xFF); + } return outSize; } @@ -750,6 +771,8 @@ int CheckEmitSourceSkip(int checkSpace, unsigned char*& pDest, int space_left_in return EmitSourceSkip(pDest, space_left_in_bank); } + space_left_in_bank -= checkSpace; + return space_left_in_bank; } @@ -768,6 +791,8 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, { // printf("LZBA Compress %d bytes\n", sourceSize); + pGlobalDictionary = pDictionary; + // Used for bank skip opcode emission int bankOffset = (int)((pDest - pDataStart) & 0xFFFF); @@ -810,14 +835,14 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, if (skipSize) { - space_left_in_bank = CheckEmitSourceSkip(2, pDest, space_left_in_bank); + int numSkips = (skipSize / MAX_STRING_SIZE) + 1; + + space_left_in_bank = CheckEmitSourceSkip(2 * numSkips, pDest, space_left_in_bank); // We need to Skip pDest += EmitSkip(pDest, skipSize); bLastEmitIsLiteral = false; lastEmittedCursorPosition = cursorPosition; - - space_left_in_bank = (int)0x10000 - (int)((pDest - pDataStart)&0xFFFF); } int tempCursorPosition = cursorPosition; @@ -830,9 +855,9 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, } else { - gapCount++; - if (gapCount >= 3) + if (gapCount > 3) break; + gapCount++; } } @@ -844,10 +869,17 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, // it from one part of the frame buffer to another part) sourceData.pData = &pSource[ cursorPosition ]; - sourceData.size = tempCursorPosition - cursorPosition + 1; + sourceData.size = tempCursorPosition - cursorPosition; - cursorPosition = tempCursorPosition; + //-------------------------- Dump, so skip dump only + space_left_in_bank = CheckEmitSourceSkip(2+sourceData.size, pDest, space_left_in_bank); + cursorPosition = AddDictionary(sourceData, cursorPosition); + + pDest += EmitLiteral(pDest, sourceData); + lastEmittedCursorPosition = cursorPosition; + + #if 0 while (sourceData.size > 0) { candidateData = LongestMatch(sourceData, dictionaryData, cursorPosition); @@ -914,6 +946,7 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, space_left_in_bank = (int)0x10000 - (int)((pDest - pDataStart)&0xFFFF); } } + #endif } else { @@ -928,6 +961,14 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, *pDest++ = 0x02; *pDest++ = 0x00; + for (int idx = 0; idx < dictionarySize; ++idx) + { + if (pSource[ idx ] != pDictionary[ idx ]) + { + assert(0); + } + } + return (int)(pDest - pOriginalDest); } diff --git a/source/lzb.h b/source/lzb.h index 7d67c09..8f43e20 100644 --- a/source/lzb.h +++ b/source/lzb.h @@ -15,7 +15,7 @@ int Old_LZB_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSiz // int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, unsigned char* pDataStart, unsigned char* pDictionary, - int dictionarySize=0); + int dictionarySize); #endif // LZB_H From 0d28c20f03f93bbe5ccb39460cf1b6efc5a9d634 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Mon, 20 Jul 2020 11:37:54 -0400 Subject: [PATCH 20/21] gsla test file, with dictionary support is decompressing correctly --- source/gsla_file.cpp | 4 ++-- source/lzb.cpp | 33 +++++++++++++++++++++------------ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/source/gsla_file.cpp b/source/gsla_file.cpp index 24b2a21..4aa4d2c 100644 --- a/source/gsla_file.cpp +++ b/source/gsla_file.cpp @@ -417,7 +417,7 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) // I don't want random data in the bank gaps, so initialize this // buffer with zero - //memset(pWorkBuffer, 0, m_frameSize * 2); + //memset(pWorkBuffer, 0xEA, m_frameSize * 2); int frameSize = LZBA_Compress(pWorkBuffer, m_pC1PixelMaps[ frameIndex ], m_frameSize, pWorkBuffer-bytes.size(), @@ -438,7 +438,7 @@ void GSLAFile::SaveToFile(const char* pFilenamePath) } // Add the RING Frame - //memset(pWorkBuffer, 0, m_frameSize * 2); + //memset(pWorkBuffer, 0xAB, m_frameSize * 2); printf("Save Ring Frame\n"); diff --git a/source/lzb.cpp b/source/lzb.cpp index 7af629c..4ce97cf 100644 --- a/source/lzb.cpp +++ b/source/lzb.cpp @@ -416,6 +416,10 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary, in } } + // Not getting better than this + if (result.size == data.size) + return result; + // This will keep us from finding matches that we can't use int dictionarySize = cursorPosition; @@ -458,13 +462,19 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary, in } } + // Not getting better than this + if (result.size == data.size) + return result; + + + #if 1 // Look for matches beyond the cursor dictionarySize = dictionary.size; if ((dictionarySize-cursorPosition) > candidate.size) { // Check the dictionary for a match, brute force - for (int dictionaryIndex = cursorPosition; dictionaryIndex <= (dictionarySize-candidate.size); ++dictionaryIndex) + for (int dictionaryIndex = cursorPosition+3; dictionaryIndex <= (dictionarySize-candidate.size); ++dictionaryIndex) { int sizeAvailable = dictionarySize - dictionaryIndex; @@ -493,7 +503,7 @@ DataString LongestMatch(const DataString& data, const DataString& dictionary, in } } } - + #endif } return result; @@ -855,7 +865,9 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, } else { - if (gapCount > 3) + // if there's a small amount of matching data, let's include + // it in the clump (try and reduce opcode emissions) + if (gapCount >= 3) break; gapCount++; } @@ -871,6 +883,7 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, sourceData.pData = &pSource[ cursorPosition ]; sourceData.size = tempCursorPosition - cursorPosition; + #if 0 // This Works //-------------------------- Dump, so skip dump only space_left_in_bank = CheckEmitSourceSkip(2+sourceData.size, pDest, space_left_in_bank); @@ -878,8 +891,8 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, pDest += EmitLiteral(pDest, sourceData); lastEmittedCursorPosition = cursorPosition; + #endif - #if 0 while (sourceData.size > 0) { candidateData = LongestMatch(sourceData, dictionaryData, cursorPosition); @@ -908,7 +921,6 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, pDest += (int)EmitReference(pDest, (int)(candidateData.pData - dictionaryData.pData), candidateData); bLastEmitIsLiteral = false; - space_left_in_bank = (int)0x10000 - (int)((pDest - pDataStart)&0xFFFF); } else if (bLastEmitIsLiteral) { @@ -919,18 +931,18 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, int space = CheckEmitSourceSkip(candidateData.size, pDest, space_left_in_bank); - if (space != space_left_in_bank) + if (space != (space_left_in_bank - candidateData.size)) { + space_left_in_bank = space-2; + // Emit a new literal pLastLiteralDest = pDest; - bLastEmitIsLiteral = true; pDest += EmitLiteral(pDest, candidateData); - - space_left_in_bank = (int)0x10000 - (int)((pDest - pDataStart)&0xFFFF); } else { // Concatenate this literal onto the previous literal + space_left_in_bank = space; pDest += ConcatLiteral(pLastLiteralDest, candidateData); } } @@ -942,11 +954,8 @@ int LZBA_Compress(unsigned char* pDest, unsigned char* pSource, int sourceSize, pLastLiteralDest = pDest; bLastEmitIsLiteral = true; pDest += EmitLiteral(pDest, candidateData); - - space_left_in_bank = (int)0x10000 - (int)((pDest - pDataStart)&0xFFFF); } } - #endif } else { From e19209dea047e08ed0dcb7cef1817be4f51b79c5 Mon Sep 17 00:00:00 2001 From: "JASON-6700K\\jandersen" Date: Wed, 22 Jul 2020 16:26:54 -0400 Subject: [PATCH 21/21] README: Updated --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index b2c49e6..4005672 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ # gsla Apple IIgs animation tool, alternative to Paintworks/$C2 animation format + +This is a C++ command line tool, that will convert a C2/Paintworks Animation +file into the more efficient GS Lzb Animation file format. + +See https://github.com/dwsJason/gslaplay, for a GSOS Sample Application +that can play these animations + +