nulib2/nufxlib/samples/TestTwirl.c
Andy McFadden 51af83986c Fix test-twirl
The test was expecting the order of threads not to change, which
was reasonable until we started throwing fake threads at it with
the recent change to "mask dataless".  We now employ a hack that
reorders the CRCs from data-class threads so that the data fork's
CRC always comes first.  Not a tremendously robust solution, but
it'll do.
2015-12-26 15:55:58 -08:00

710 lines
19 KiB
C

/*
* NuFX archive manipulation library
* Copyright (C) 2000-2007 by Andy McFadden, All Rights Reserved.
* This is free software; you can redistribute it and/or modify it under the
* terms of the BSD License, see the file COPYING.LIB.
*
* Recompress records in place, several times, possibly deleting records
* or threads as we go. The goal is to perform a large number of operations
* that modify the archive without closing and reopening it.
*
* Depending on which #defines are enabled, this can be very destructive,
* so a copy of the archive is made before processing begins.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "NufxLib.h"
#include "Common.h"
/* copy the archive to this file before starting */
static const char* kWorkFileName = "TwirlCopy678";
static const char* kTempFileName = "TwirlTmp789";
/* after loading this much stuff into memory, flush changes */
const int kMaxHeldLen = 1024 * 1024;
/*
* A list of CRCs.
*/
typedef struct CRCList {
int numEntries;
uint16_t* entries;
} CRCList;
/*
* Returns true if the compression type is supported, false otherwise.
*/
int CompressionSupported(NuValue compression)
{
int result;
switch (compression) {
case kNuCompressNone:
result = true;
break;
case kNuCompressSQ:
result = (NuTestFeature(kNuFeatureCompressSQ) == kNuErrNone);
break;
case kNuCompressLZW1:
case kNuCompressLZW2:
result = (NuTestFeature(kNuFeatureCompressLZW) == kNuErrNone);
break;
case kNuCompressLZC12:
case kNuCompressLZC16:
result = (NuTestFeature(kNuFeatureCompressLZC) == kNuErrNone);
break;
case kNuCompressDeflate:
result = (NuTestFeature(kNuFeatureCompressDeflate) == kNuErrNone);
break;
case kNuCompressBzip2:
result = (NuTestFeature(kNuFeatureCompressBzip2) == kNuErrNone);
break;
default:
assert(false);
result = false;
}
/*printf("Returning %d for %ld\n", result, compression);*/
return result;
}
/*
* This gets called when a buffer DataSource is no longer needed.
*/
NuResult FreeCallback(NuArchive* pArchive, void* args)
{
free(args);
return kNuOK;
}
/*
* Dump a CRC list.
*/
void DumpCRCs(const CRCList* pCRCList)
{
int i;
printf(" NumEntries: %d\n", pCRCList->numEntries);
for (i = 0; i < pCRCList->numEntries; i++)
printf(" %5d: 0x%04x\n", i, pCRCList->entries[i]);
}
/*
* Free a CRC list.
*/
void FreeCRCs(CRCList* pCRCList)
{
if (pCRCList == NULL)
return;
free(pCRCList->entries);
free(pCRCList);
}
/*
* Gather a list of CRCs from the archive.
*
* We assume there are at most two data threads (e.g. data fork and rsrc
* fork) in a record.
*
* Returns the list on success, NULL on failure.
*/
CRCList* GatherCRCs(NuArchive* pArchive)
{
NuError err = kNuErrNone;
const NuMasterHeader* pMasterHeader;
CRCList* pCRCList = NULL;
uint16_t* pEntries = NULL;
long recCount, maxCRCs;
long recIdx, crcIdx;
int i;
pCRCList = malloc(sizeof(*pCRCList));
if (pCRCList == NULL) {
fprintf(stderr, "ERROR: couldn't alloc CRC list\n");
err = kNuErrGeneric;
goto bail;
}
memset(pCRCList, 0, sizeof(*pCRCList));
/* get record count out of master header, just for fun */
err = NuGetMasterHeader(pArchive, &pMasterHeader);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: couldn't get master header (err=%d)\n", err);
goto bail;
}
recCount = pMasterHeader->mhTotalRecords;
maxCRCs = recCount * 2;
pEntries = malloc(sizeof(*pEntries) * maxCRCs);
if (pEntries == NULL) {
fprintf(stderr, "ERROR: unable to alloc CRC list (%ld entries)\n",
maxCRCs);
err = kNuErrGeneric;
goto bail;
}
pCRCList->entries = pEntries;
for (i = 0; i < maxCRCs; i++)
pEntries[i] = 0xdead;
/*
* Enumerate our way through the records. If something was disturbed
* we should end up in a different place and the CRCs will be off.
*/
crcIdx = 0;
for (recIdx = 0; recIdx < recCount; recIdx++) {
NuRecordIdx recordIdx;
const NuRecord* pRecord;
const NuThread* pThread;
err = NuGetRecordIdxByPosition(pArchive, recIdx, &recordIdx);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: couldn't get record #%ld (err=%d)\n",
recIdx, err);
goto bail;
}
err = NuGetRecord(pArchive, recordIdx, &pRecord);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to get recordIdx %u\n", recordIdx);
goto bail;
}
if (NuRecordGetNumThreads(pRecord) == 0) {
fprintf(stderr, "ERROR: not expecting empty record (%u)!\n",
recordIdx);
err = kNuErrGeneric;
goto bail;
}
int rsrcCrcIdx = -1;
for (i = 0; i < (int)NuRecordGetNumThreads(pRecord); i++) {
pThread = NuGetThread(pRecord, i);
if (pThread->thThreadClass == kNuThreadClassData) {
if (crcIdx >= maxCRCs) {
fprintf(stderr, "ERROR: CRC buffer exceeded\n");
assert(false);
err = kNuErrGeneric;
goto bail;
}
/*
* Ensure that the data fork CRC comes first. Otherwise
* we can fail if it gets rearranged. This is only a
* problem for GSHK-created archives that don't have
* threads for every fork, so "mask dataless" is create
* fake entries.
*
* The correct way to do this is to store a tuple
* { thread-kind, crc }, but that's more work.
*/
if (pThread->thThreadKind == kNuThreadKindRsrcFork) {
rsrcCrcIdx = crcIdx;
}
if (pThread->thThreadKind == kNuThreadKindDataFork &&
rsrcCrcIdx != -1)
{
/* this is the data fork, we've already seen the
resource fork; swap entries */
pEntries[crcIdx++] = pEntries[rsrcCrcIdx];
pEntries[rsrcCrcIdx] = pThread->thThreadCRC;
} else {
pEntries[crcIdx++] = pThread->thThreadCRC;
}
}
}
}
pCRCList->numEntries = crcIdx;
DumpCRCs(pCRCList);
bail:
if (err != kNuErrNone) {
FreeCRCs(pCRCList);
pCRCList = NULL;
}
return pCRCList;
}
/*
* Compare the current set of CRCs against our saved list. If any of
* the records or threads were deleted or rearranged, this will fail.
* I happen to think this is a *good* thing: if something is the least
* bit screwy, I want to know about it.
*
* Unfortunately, if we *deliberately* delete records, this can't
* help us with the survivors.
*
* Returns 0 on success, nonzero on failure.
*/
int CompareCRCs(NuArchive* pArchive, const CRCList* pOldCRCList)
{
CRCList* pNewCRCList = NULL;
int result = -1;
int badCrc = 0;
int i;
pNewCRCList = GatherCRCs(pArchive);
if (pNewCRCList == NULL) {
fprintf(stderr, "ERROR: unable to gather new list\n");
goto bail;
}
if (pOldCRCList->numEntries != pNewCRCList->numEntries) {
fprintf(stderr, "ERROR: numEntries mismatch: %d vs %d\n",
pOldCRCList->numEntries, pNewCRCList->numEntries);
goto bail;
}
for (i = 0; i < pNewCRCList->numEntries; i++) {
if (pOldCRCList->entries[i] != pNewCRCList->entries[i]) {
fprintf(stderr, "ERROR: CRC mismatch: %5d old=0x%04x new=0x%04x\n",
i, pOldCRCList->entries[i], pNewCRCList->entries[i]);
badCrc = 1;
}
}
if (!badCrc) {
printf(" Matched %d CRCs\n", pOldCRCList->numEntries);
result = 0;
}
bail:
FreeCRCs(pNewCRCList);
return result;
}
/*
* Recompress a single thread.
*
* This entails (1) extracting the existing thread, (2) deleting the
* thread, and (3) adding the extracted data.
*
* All of this good stuff gets queued up until the next NuFlush call.
*/
NuError RecompressThread(NuArchive* pArchive, const NuRecord* pRecord,
const NuThread* pThread)
{
NuError err = kNuErrNone;
NuDataSource* pDataSource = NULL;
NuDataSink* pDataSink = NULL;
uint8_t* buf = NULL;
if (pThread->actualThreadEOF == 0) {
buf = malloc(1);
if (buf == NULL) {
fprintf(stderr, "ERROR: failed allocating trivial buffer\n");
err = kNuErrGeneric;
goto bail;
}
} else {
/*
* Create a buffer and data sink to hold the data.
*/
buf = malloc(pThread->actualThreadEOF);
if (buf == NULL) {
fprintf(stderr, "ERROR: failed allocating %u bytes\n",
pThread->actualThreadEOF);
err = kNuErrGeneric;
goto bail;
}
err = NuCreateDataSinkForBuffer(true, kNuConvertOff, buf,
pThread->actualThreadEOF, &pDataSink);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to create data sink (err=%d)\n",err);
goto bail;
}
/*
* Extract the data.
*/
err = NuExtractThread(pArchive, pThread->threadIdx, pDataSink);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: failed extracting thread %u in '%s': %s\n",
pThread->threadIdx, pRecord->filenameMOR, NuStrError(err));
goto bail;
}
}
/*
* Delete the existing thread.
*/
err = NuDeleteThread(pArchive, pThread->threadIdx);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to delete thread %u\n",
pThread->threadIdx);
goto bail;
}
/*
* Create a data source for the new thread. Specify a callback to free
* the buffer when NufxLib is done with it.
*/
err = NuCreateDataSourceForBuffer(kNuThreadFormatUncompressed,
0, buf, 0, pThread->actualThreadEOF, FreeCallback, &pDataSource);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to create data source (err=%d)\n", err);
goto bail;
}
buf = NULL;
/*
* Create replacement thread.
*/
err = NuAddThread(pArchive, pRecord->recordIdx, NuGetThreadID(pThread),
pDataSource, NULL);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to add new thread ID=0x%08x (err=%d)\n",
NuGetThreadID(pThread), err);
goto bail;
}
pDataSource = NULL; /* now owned by NufxLib */
bail:
NuFreeDataSink(pDataSink);
NuFreeDataSource(pDataSource);
free(buf);
return err;
}
/*
* Recompress a single record.
*
* The amount of data we're holding in memory as a result of the
* recompression is placed in "*pLen".
*/
NuError RecompressRecord(NuArchive* pArchive, NuRecordIdx recordIdx, long* pLen)
{
NuError err = kNuErrNone;
const NuRecord* pRecord;
const NuThread* pThread;
int i;
printf(" Recompressing %u\n", recordIdx);
*pLen = 0;
err = NuGetRecord(pArchive, recordIdx, &pRecord);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to get record %u (err=%d)\n",
recordIdx, err);
goto bail;
}
for (i = 0; i < (int)NuRecordGetNumThreads(pRecord); i++) {
pThread = NuGetThread(pRecord, i);
if (pThread->thThreadClass == kNuThreadClassData) {
/*printf(" Recompressing %d (threadID=0x%08lx)\n", i,
NuGetThreadID(pThread));*/
err = RecompressThread(pArchive, pRecord, pThread);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: failed recompressing thread %u "
" in record %u (err=%d)\n",
pThread->threadIdx, pRecord->recordIdx, err);
goto bail;
}
*pLen += pThread->actualThreadEOF;
} else {
/*printf(" Skipping %d (threadID=0x%08lx)\n", i,
NuGetThreadID(pThread));*/
}
}
bail:
return err;
}
/*
* Recompress every data thread in the archive.
*/
NuError RecompressArchive(NuArchive* pArchive, NuValue compression)
{
NuError err = kNuErrNone;
NuRecordIdx* pIndices = NULL;
NuAttr countAttr;
long heldLen;
long idx;
err = NuSetValue(pArchive, kNuValueDataCompression, compression);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to set compression to %u (err=%d)\n",
compression, err);
goto bail;
}
printf("Recompressing threads with compression type %u\n", compression);
err = NuGetAttr(pArchive, kNuAttrNumRecords, &countAttr);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to get numRecords (err=%d)\n", err);
goto bail;
}
if (countAttr == 0) {
printf("No records found!\n");
goto bail;
}
/*
* Get all of the indices up front. This way, if something causes a
* record to "disappear" during processing, we will know about it.
*/
pIndices = malloc(countAttr * sizeof(*pIndices));
if (pIndices == NULL) {
fprintf(stderr, "ERROR: malloc on %u indices failed\n", countAttr);
err = kNuErrGeneric;
goto bail;
}
for (idx = 0; idx < (int)countAttr; idx++) {
err = NuGetRecordIdxByPosition(pArchive, idx, &pIndices[idx]);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: couldn't get record #%ld (err=%d)\n",
idx, err);
goto bail;
}
}
/*
* Walk through the index list, handling each record individually.
*/
heldLen = 0;
for (idx = 0; idx < (int)countAttr; idx++) {
long recHeldLen;
err = RecompressRecord(pArchive, pIndices[idx], &recHeldLen);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: failed recompressing record %u (err=%d)\n",
pIndices[idx], err);
goto bail;
}
heldLen += recHeldLen;
if (heldLen > kMaxHeldLen) {
uint32_t statusFlags;
printf(" (flush)\n");
err = NuFlush(pArchive, &statusFlags);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: intra-recompress flush failed: %s\n",
NuStrError(err));
goto bail;
}
heldLen = 0;
}
}
bail:
free(pIndices);
return err;
}
/*
* Initiate the twirling.
*/
int TwirlArchive(const char* filename)
{
NuError err = kNuErrNone;
NuArchive* pArchive = NULL;
CRCList* pCRCList = NULL;
int compression;
int cc;
/*
* Open the archive after removing any temp file remnants.
*/
cc = unlink(kTempFileName);
if (cc == 0)
printf("Removed stale temp file '%s'\n", kTempFileName);
err = NuOpenRW(filename, kTempFileName, 0, &pArchive);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to open archive '%s': %s\n",
filename, NuStrError(err));
goto bail;
}
/*
* Mask records with no data threads, so we don't have to
* special-case them.
*/
err = NuSetValue(pArchive, kNuValueMaskDataless, true);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: couldn't mask dataless (err=%d)\n", err);
goto bail;
}
pCRCList = GatherCRCs(pArchive);
if (pCRCList == NULL) {
fprintf(stderr, "ERROR: unable to get CRC list\n");
goto bail;
}
/*
* For each type of compression, recompress the entire archive.
*/
for (compression = kNuCompressNone; compression <= kNuCompressBzip2;
compression++)
{
uint32_t statusFlags;
if (!CompressionSupported(compression))
continue;
err = RecompressArchive(pArchive, compression);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: recompress failed: %s\n", NuStrError(err));
goto bail;
}
err = NuFlush(pArchive, &statusFlags);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: post-recompress flush failed: %s\n",
NuStrError(err));
goto bail;
}
}
/*
* Same thing, reverse order. We want to start with the same one we
* ended on above, so we can practice skipping over things.
*/
for (compression = kNuCompressBzip2; compression >= kNuCompressNone;
compression--)
{
uint32_t statusFlags;
if (!CompressionSupported(compression))
continue;
err = RecompressArchive(pArchive, compression);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: recompress2 failed: %s\n", NuStrError(err));
goto bail;
}
err = NuFlush(pArchive, &statusFlags);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: post-recompress flush2 failed: %s\n",
NuStrError(err));
goto bail;
}
}
if (CompareCRCs(pArchive, pCRCList) != 0) {
fprintf(stderr, "ERROR: CRCs didn't match\n");
goto bail;
}
printf("Done!\n");
bail:
FreeCRCs(pCRCList);
if (pArchive != NULL) {
NuAbort(pArchive);
NuClose(pArchive);
}
return (err != kNuErrNone);
}
/*
* Copy from the current offset in "srcfp" to a new file called
* "outFileName". Returns a writable file descriptor for the new file
* on success, or NULL on error.
*
* (Note "CopyFile()" exists under Win32.)
*/
FILE* MyCopyFile(const char* outFileName, FILE* srcfp)
{
char buf[24576];
FILE* outfp;
size_t count;
outfp = fopen(outFileName, kNuFileOpenWriteTrunc);
if (outfp == NULL) {
fprintf(stderr, "ERROR: unable to open '%s' (err=%d)\n", outFileName,
errno);
return NULL;
}
while (!feof(srcfp)) {
count = fread(buf, 1, sizeof(buf), srcfp);
if (count == 0)
break;
if (fwrite(buf, 1, count, outfp) != count) {
fprintf(stderr, "ERROR: failed writing outfp (err=%d)\n", errno);
fclose(outfp);
return NULL;
}
}
if (ferror(srcfp)) {
fprintf(stderr, "ERROR: failed reading srcfp (err=%d)\n", errno);
fclose(outfp);
return NULL;
}
return outfp;
}
/*
* Let's get started.
*/
int main(int argc, char** argv)
{
int32_t major, minor, bug;
const char* pBuildDate;
FILE* srcfp = NULL;
FILE* infp = NULL;
int cc;
/* don't buffer output */
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
(void) NuGetVersion(&major, &minor, &bug, &pBuildDate, NULL);
printf("Using NuFX lib %d.%d.%d built on or after %s\n\n",
major, minor, bug, pBuildDate);
if (argc == 2) {
srcfp = fopen(argv[1], kNuFileOpenReadOnly);
if (srcfp == NULL) {
perror("fopen failed");
exit(1);
}
} else {
fprintf(stderr, "ERROR: you have to specify a filename\n");
exit(2);
}
printf("Copying '%s' to '%s'\n", argv[1], kWorkFileName);
infp = MyCopyFile(kWorkFileName, srcfp);
if (infp == NULL) {
fprintf(stderr, "Copy failed, bailing.\n");
exit(1);
}
fclose(srcfp);
fclose(infp);
cc = TwirlArchive(kWorkFileName);
exit(cc != 0);
}