nulib2/nufxlib/samples/TestTwirl.c
Andy McFadden e2088e64d3 Distinguish Unicode and Mac OS Roman strings
NufxLib has historically made no effort to distinguish between
the character set used for filenames on the local disk, and for
filenames stored within the archive.  Now all Unicode filename
strings use the UNICHAR type and have "UNI" in the name, and all
Mac OS Roman strings have "MOR" in the name.  (The naming
convention makes it obvious when you're assigning the wrong thing;
on Linux both formats are char*, so the compiler won't tell you
if you get it wrong.)

The distinction is necessary because filesystems generally support
Unicode these days, but on Windows you need to use a separate
set of wide-character file I/O functions.  (On Linux it all works
with "narrow" strings, and the UTF-8 encoding is interpreted by
applications.)  The character set used for NuFX archive filenames
is MOR, matching what GS/OS + HFS supported, and we want to be able
to convert back and forth between MOR and a Unicode representation.

This change updates the various character types and string names,
adds conversion functions, and updates NuLib2 for proper execution
on Linux.  It does not include the (probably extensive) changes
required for Windows UTF-16 support.  Instead, the conversion
functions are no-ops, which should result in NuLib2 for Windows
continuing to behave in the same slightly broken way.

This adds "test-names", which exercises Unicode filenames a bit.
It will not pass on Win32.

Also, tweaked the Linux makefiles to have explicit dependencies,
rather than empty space and an expectation that "makedepend" exists.

Also, minor source code cleanups.

While this probably doesn't affect binary compatibility -- it's
mainly a matter of naming and string interpretation -- there's
enough going on that it should be considered an API revision, so
this updates the version to 3.0.0.
2015-01-02 17:14:34 -08:00

686 lines
18 KiB
C

/*
* NuFX archive manipulation library
* Copyright (C) 2000-2007 by Andy McFadden, All Rights Reserved.
* This is free software; you can redistribute it and/or modify it under the
* terms of the BSD License, see the file COPYING.LIB.
*
* Recompress records in place, several times, possibly deleting records
* or threads as we go. The goal is to perform a large number of operations
* that modify the archive without closing and reopening it.
*
* Depending on which #defines are enabled, this can be very destructive,
* so a copy of the archive is made before processing begins.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "NufxLib.h"
#include "Common.h"
/* copy the archive to this file before starting */
static const char* kWorkFileName = "TwirlCopy678";
static const char* kTempFileName = "TwirlTmp789";
/* after loading this much stuff into memory, flush changes */
const int kMaxHeldLen = 1024 * 1024;
/*
* A list of CRCs.
*/
typedef struct CRCList {
int numEntries;
uint16_t* entries;
} CRCList;
/*
* Returns true if the compression type is supported, false otherwise.
*/
int CompressionSupported(NuValue compression)
{
int result;
switch (compression) {
case kNuCompressNone:
result = true;
break;
case kNuCompressSQ:
result = (NuTestFeature(kNuFeatureCompressSQ) == kNuErrNone);
break;
case kNuCompressLZW1:
case kNuCompressLZW2:
result = (NuTestFeature(kNuFeatureCompressLZW) == kNuErrNone);
break;
case kNuCompressLZC12:
case kNuCompressLZC16:
result = (NuTestFeature(kNuFeatureCompressLZC) == kNuErrNone);
break;
case kNuCompressDeflate:
result = (NuTestFeature(kNuFeatureCompressDeflate) == kNuErrNone);
break;
case kNuCompressBzip2:
result = (NuTestFeature(kNuFeatureCompressBzip2) == kNuErrNone);
break;
default:
assert(false);
result = false;
}
/*printf("Returning %d for %ld\n", result, compression);*/
return result;
}
/*
* This gets called when a buffer DataSource is no longer needed.
*/
NuResult FreeCallback(NuArchive* pArchive, void* args)
{
free(args);
return kNuOK;
}
/*
* Dump a CRC list.
*/
void DumpCRCs(const CRCList* pCRCList)
{
int i;
printf(" NumEntries: %d\n", pCRCList->numEntries);
for (i = 0; i < pCRCList->numEntries; i++)
printf(" %5d: 0x%04x\n", i, pCRCList->entries[i]);
}
/*
* Free a CRC list.
*/
void FreeCRCs(CRCList* pCRCList)
{
if (pCRCList == NULL)
return;
free(pCRCList->entries);
free(pCRCList);
}
/*
* Gather a list of CRCs from the archive.
*
* We assume there are at most two data threads (e.g. data fork and rsrc
* fork) in a record.
*
* Returns the list on success, NULL on failure.
*/
CRCList* GatherCRCs(NuArchive* pArchive)
{
NuError err = kNuErrNone;
const NuMasterHeader* pMasterHeader;
CRCList* pCRCList = NULL;
uint16_t* pEntries = NULL;
long recCount, maxCRCs;
long recIdx, crcIdx;
int i;
pCRCList = malloc(sizeof(*pCRCList));
if (pCRCList == NULL) {
fprintf(stderr, "ERROR: couldn't alloc CRC list\n");
err = kNuErrGeneric;
goto bail;
}
memset(pCRCList, 0, sizeof(*pCRCList));
/* get record count out of master header, just for fun */
err = NuGetMasterHeader(pArchive, &pMasterHeader);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: couldn't get master header (err=%d)\n", err);
goto bail;
}
recCount = pMasterHeader->mhTotalRecords;
maxCRCs = recCount * 2;
pEntries = malloc(sizeof(*pEntries) * maxCRCs);
if (pEntries == NULL) {
fprintf(stderr, "ERROR: unable to alloc CRC list (%ld entries)\n",
maxCRCs);
err = kNuErrGeneric;
goto bail;
}
pCRCList->entries = pEntries;
for (i = 0; i < maxCRCs; i++)
pEntries[i] = 0xdead;
/*
* Enumerate our way through the records. If something was disturbed
* we should end up in a different place and the CRCs will be off.
*/
crcIdx = 0;
for (recIdx = 0; recIdx < recCount; recIdx++) {
NuRecordIdx recordIdx;
const NuRecord* pRecord;
const NuThread* pThread;
err = NuGetRecordIdxByPosition(pArchive, recIdx, &recordIdx);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: couldn't get record #%ld (err=%d)\n",
recIdx, err);
goto bail;
}
err = NuGetRecord(pArchive, recordIdx, &pRecord);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to get recordIdx %u\n", recordIdx);
goto bail;
}
if (NuRecordGetNumThreads(pRecord) == 0) {
fprintf(stderr, "ERROR: not expecting empty record (%u)!\n",
recordIdx);
err = kNuErrGeneric;
goto bail;
}
for (i = 0; i < (int)NuRecordGetNumThreads(pRecord); i++) {
pThread = NuGetThread(pRecord, i);
if (pThread->thThreadClass == kNuThreadClassData) {
if (crcIdx >= maxCRCs) {
fprintf(stderr, "ERROR: CRC buffer exceeded\n");
assert(false);
err = kNuErrGeneric;
goto bail;
}
pEntries[crcIdx++] = pThread->thThreadCRC;
}
}
}
pCRCList->numEntries = crcIdx;
DumpCRCs(pCRCList);
bail:
if (err != kNuErrNone) {
FreeCRCs(pCRCList);
pCRCList = NULL;
}
return pCRCList;
}
/*
* Compare the current set of CRCs against our saved list. If any of
* the records or threads were deleted or rearranged, this will fail.
* I happen to think this is a *good* thing: if something is the least
* bit screwy, I want to know about it.
*
* Unfortunately, if we *deliberately* delete records, this can't
* help us with the survivors.
*
* Returns 0 on success, nonzero on failure.
*/
int CompareCRCs(NuArchive* pArchive, const CRCList* pOldCRCList)
{
CRCList* pNewCRCList = NULL;
int result = -1;
int badCrc = 0;
int i;
pNewCRCList = GatherCRCs(pArchive);
if (pNewCRCList == NULL) {
fprintf(stderr, "ERROR: unable to gather new list\n");
goto bail;
}
if (pOldCRCList->numEntries != pNewCRCList->numEntries) {
fprintf(stderr, "ERROR: numEntries mismatch: %d vs %d\n",
pOldCRCList->numEntries, pNewCRCList->numEntries);
goto bail;
}
for (i = 0; i < pNewCRCList->numEntries; i++) {
if (pOldCRCList->entries[i] != pNewCRCList->entries[i]) {
fprintf(stderr, "ERROR: CRC mismatch: %5d old=0x%04x new=0x%04x\n",
i, pOldCRCList->entries[i], pNewCRCList->entries[i]);
badCrc = 1;
}
}
if (!badCrc) {
printf(" Matched %d CRCs\n", pOldCRCList->numEntries);
result = 0;
}
bail:
FreeCRCs(pNewCRCList);
return result;
}
/*
* Recompress a single thread.
*
* This entails (1) extracting the existing thread, (2) deleting the
* thread, and (3) adding the extracted data.
*
* All of this good stuff gets queued up until the next NuFlush call.
*/
NuError RecompressThread(NuArchive* pArchive, const NuRecord* pRecord,
const NuThread* pThread)
{
NuError err = kNuErrNone;
NuDataSource* pDataSource = NULL;
NuDataSink* pDataSink = NULL;
uint8_t* buf = NULL;
if (pThread->actualThreadEOF == 0) {
buf = malloc(1);
if (buf == NULL) {
fprintf(stderr, "ERROR: failed allocating trivial buffer\n");
err = kNuErrGeneric;
goto bail;
}
} else {
/*
* Create a buffer and data sink to hold the data.
*/
buf = malloc(pThread->actualThreadEOF);
if (buf == NULL) {
fprintf(stderr, "ERROR: failed allocating %u bytes\n",
pThread->actualThreadEOF);
err = kNuErrGeneric;
goto bail;
}
err = NuCreateDataSinkForBuffer(true, kNuConvertOff, buf,
pThread->actualThreadEOF, &pDataSink);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to create data sink (err=%d)\n",err);
goto bail;
}
/*
* Extract the data.
*/
err = NuExtractThread(pArchive, pThread->threadIdx, pDataSink);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: failed extracting thread %u in '%s': %s\n",
pThread->threadIdx, pRecord->filenameMOR, NuStrError(err));
goto bail;
}
}
/*
* Delete the existing thread.
*/
err = NuDeleteThread(pArchive, pThread->threadIdx);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to delete thread %u\n",
pThread->threadIdx);
goto bail;
}
/*
* Create a data source for the new thread. Specify a callback to free
* the buffer when NufxLib is done with it.
*/
err = NuCreateDataSourceForBuffer(kNuThreadFormatUncompressed,
0, buf, 0, pThread->actualThreadEOF, FreeCallback, &pDataSource);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to create data source (err=%d)\n", err);
goto bail;
}
buf = NULL;
/*
* Create replacement thread.
*/
err = NuAddThread(pArchive, pRecord->recordIdx, NuGetThreadID(pThread),
pDataSource, NULL);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to add new thread ID=0x%08x (err=%d)\n",
NuGetThreadID(pThread), err);
goto bail;
}
pDataSource = NULL; /* now owned by NufxLib */
bail:
NuFreeDataSink(pDataSink);
NuFreeDataSource(pDataSource);
free(buf);
return err;
}
/*
* Recompress a single record.
*
* The amount of data we're holding in memory as a result of the
* recompression is placed in "*pLen".
*/
NuError RecompressRecord(NuArchive* pArchive, NuRecordIdx recordIdx, long* pLen)
{
NuError err = kNuErrNone;
const NuRecord* pRecord;
const NuThread* pThread;
int i;
printf(" Recompressing %u\n", recordIdx);
*pLen = 0;
err = NuGetRecord(pArchive, recordIdx, &pRecord);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to get record %u (err=%d)\n",
recordIdx, err);
goto bail;
}
for (i = 0; i < (int)NuRecordGetNumThreads(pRecord); i++) {
pThread = NuGetThread(pRecord, i);
if (pThread->thThreadClass == kNuThreadClassData) {
/*printf(" Recompressing %d (threadID=0x%08lx)\n", i,
NuGetThreadID(pThread));*/
err = RecompressThread(pArchive, pRecord, pThread);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: failed recompressing thread %u "
" in record %u (err=%d)\n",
pThread->threadIdx, pRecord->recordIdx, err);
goto bail;
}
*pLen += pThread->actualThreadEOF;
} else {
/*printf(" Skipping %d (threadID=0x%08lx)\n", i,
NuGetThreadID(pThread));*/
}
}
bail:
return err;
}
/*
* Recompress every data thread in the archive.
*/
NuError RecompressArchive(NuArchive* pArchive, NuValue compression)
{
NuError err = kNuErrNone;
NuRecordIdx* pIndices = NULL;
NuAttr countAttr;
long heldLen;
long idx;
err = NuSetValue(pArchive, kNuValueDataCompression, compression);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to set compression to %u (err=%d)\n",
compression, err);
goto bail;
}
printf("Recompressing threads with compression type %u\n", compression);
err = NuGetAttr(pArchive, kNuAttrNumRecords, &countAttr);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to get numRecords (err=%d)\n", err);
goto bail;
}
if (countAttr == 0) {
printf("No records found!\n");
goto bail;
}
/*
* Get all of the indices up front. This way, if something causes a
* record to "disappear" during processing, we will know about it.
*/
pIndices = malloc(countAttr * sizeof(*pIndices));
if (pIndices == NULL) {
fprintf(stderr, "ERROR: malloc on %u indices failed\n", countAttr);
err = kNuErrGeneric;
goto bail;
}
for (idx = 0; idx < (int)countAttr; idx++) {
err = NuGetRecordIdxByPosition(pArchive, idx, &pIndices[idx]);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: couldn't get record #%ld (err=%d)\n",
idx, err);
goto bail;
}
}
/*
* Walk through the index list, handling each record individually.
*/
heldLen = 0;
for (idx = 0; idx < (int)countAttr; idx++) {
long recHeldLen;
err = RecompressRecord(pArchive, pIndices[idx], &recHeldLen);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: failed recompressing record %u (err=%d)\n",
pIndices[idx], err);
goto bail;
}
heldLen += recHeldLen;
if (heldLen > kMaxHeldLen) {
uint32_t statusFlags;
printf(" (flush)\n");
err = NuFlush(pArchive, &statusFlags);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: intra-recompress flush failed: %s\n",
NuStrError(err));
goto bail;
}
heldLen = 0;
}
}
bail:
free(pIndices);
return err;
}
/*
* Initiate the twirling.
*/
int TwirlArchive(const char* filename)
{
NuError err = kNuErrNone;
NuArchive* pArchive = NULL;
CRCList* pCRCList = NULL;
int compression;
int cc;
/*
* Open the archive after removing any temp file remnants.
*/
cc = unlink(kTempFileName);
if (cc == 0)
printf("Removed stale temp file '%s'\n", kTempFileName);
err = NuOpenRW(filename, kTempFileName, 0, &pArchive);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: unable to open archive '%s': %s\n",
filename, NuStrError(err));
goto bail;
}
/*
* Mask records with no data threads, so we don't have to
* special-case them.
*/
err = NuSetValue(pArchive, kNuValueMaskDataless, true);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: couldn't mask dataless (err=%d)\n", err);
goto bail;
}
pCRCList = GatherCRCs(pArchive);
if (pCRCList == NULL) {
fprintf(stderr, "ERROR: unable to get CRC list\n");
goto bail;
}
/*
* For each type of compression, recompress the entire archive.
*/
for (compression = kNuCompressNone; compression <= kNuCompressBzip2;
compression++)
{
uint32_t statusFlags;
if (!CompressionSupported(compression))
continue;
err = RecompressArchive(pArchive, compression);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: recompress failed: %s\n", NuStrError(err));
goto bail;
}
err = NuFlush(pArchive, &statusFlags);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: post-recompress flush failed: %s\n",
NuStrError(err));
goto bail;
}
}
/*
* Same thing, reverse order. We want to start with the same one we
* ended on above, so we can practice skipping over things.
*/
for (compression = kNuCompressBzip2; compression >= kNuCompressNone;
compression--)
{
uint32_t statusFlags;
if (!CompressionSupported(compression))
continue;
err = RecompressArchive(pArchive, compression);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: recompress2 failed: %s\n", NuStrError(err));
goto bail;
}
err = NuFlush(pArchive, &statusFlags);
if (err != kNuErrNone) {
fprintf(stderr, "ERROR: post-recompress flush2 failed: %s\n",
NuStrError(err));
goto bail;
}
}
if (CompareCRCs(pArchive, pCRCList) != 0) {
fprintf(stderr, "ERROR: CRCs didn't match\n");
goto bail;
}
printf("Done!\n");
bail:
FreeCRCs(pCRCList);
if (pArchive != NULL) {
NuAbort(pArchive);
NuClose(pArchive);
}
return (err != kNuErrNone);
}
/*
* Copy from the current offset in "srcfp" to a new file called
* "outFileName". Returns a writable file descriptor for the new file
* on success, or NULL on error.
*
* (Note "CopyFile()" exists under Win32.)
*/
FILE* MyCopyFile(const char* outFileName, FILE* srcfp)
{
char buf[24576];
FILE* outfp;
size_t count;
outfp = fopen(outFileName, kNuFileOpenWriteTrunc);
if (outfp == NULL) {
fprintf(stderr, "ERROR: unable to open '%s' (err=%d)\n", outFileName,
errno);
return NULL;
}
while (!feof(srcfp)) {
count = fread(buf, 1, sizeof(buf), srcfp);
if (count == 0)
break;
if (fwrite(buf, 1, count, outfp) != count) {
fprintf(stderr, "ERROR: failed writing outfp (err=%d)\n", errno);
fclose(outfp);
return NULL;
}
}
if (ferror(srcfp)) {
fprintf(stderr, "ERROR: failed reading srcfp (err=%d)\n", errno);
fclose(outfp);
return NULL;
}
return outfp;
}
/*
* Let's get started.
*/
int main(int argc, char** argv)
{
int32_t major, minor, bug;
const char* pBuildDate;
FILE* srcfp = NULL;
FILE* infp = NULL;
int cc;
/* don't buffer output */
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
(void) NuGetVersion(&major, &minor, &bug, &pBuildDate, NULL);
printf("Using NuFX lib %d.%d.%d built on or after %s\n\n",
major, minor, bug, pBuildDate);
if (argc == 2) {
srcfp = fopen(argv[1], kNuFileOpenReadOnly);
if (srcfp == NULL) {
perror("fopen failed");
exit(1);
}
} else {
fprintf(stderr, "ERROR: you have to specify a filename\n");
exit(2);
}
printf("Copying '%s' to '%s'\n", argv[1], kWorkFileName);
infp = MyCopyFile(kWorkFileName, srcfp);
if (infp == NULL) {
fprintf(stderr, "Copy failed, bailing.\n");
exit(1);
}
fclose(srcfp);
fclose(infp);
cc = TwirlArchive(kWorkFileName);
exit(cc != 0);
}