diff options
Diffstat (limited to 'programs/lz4io.c')
| -rw-r--r-- | programs/lz4io.c | 898 |
1 files changed, 700 insertions, 198 deletions
diff --git a/programs/lz4io.c b/programs/lz4io.c index 28d6537..a274798 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -53,11 +53,11 @@ #include <time.h> /* clock */ #include <sys/types.h> /* stat64 */ #include <sys/stat.h> /* stat64 */ -#include "lz4io.h" #include "lz4.h" /* still required for legacy format */ #include "lz4hc.h" /* still required for legacy format */ #define LZ4F_STATIC_LINKING_ONLY #include "lz4frame.h" +#include "lz4io.h" /***************************** @@ -90,6 +90,7 @@ * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ @@ -102,23 +103,29 @@ static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result } } static const clock_t refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_time = 0; +#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ /************************************** * Local Parameters **************************************/ -static int g_overwrite = 1; -static int g_testMode = 0; -static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; -static int g_blockChecksum = 0; -static int g_streamChecksum = 1; -static int g_blockIndependence = 1; -static int g_sparseFileSupport = 1; -static int g_contentSizeFlag = 0; -static int g_useDictionary = 0; -static unsigned g_favorDecSpeed = 0; -static const char* g_dictionaryFilename = NULL; +struct LZ4IO_prefs_s { + int passThrough; + int overwrite; + int testMode; + int blockSizeId; + size_t blockSize; + int blockChecksum; + int streamChecksum; + int blockIndependence; + int sparseFileSupport; + int contentSizeFlag; + int useDictionary; + unsigned favorDecSpeed; + const char* dictionaryFilename; + int removeSrcFile; +}; /************************************** * Exceptions @@ -150,55 +157,110 @@ static const char* g_dictionaryFilename = NULL; /* ****************** Parameters ******************** */ /* ************************************************** */ -int LZ4IO_setDictionaryFilename(const char* dictionaryFilename) { - g_dictionaryFilename = dictionaryFilename; - g_useDictionary = dictionaryFilename != NULL; - return g_useDictionary; +LZ4IO_prefs_t* LZ4IO_defaultPreferences(void) +{ + LZ4IO_prefs_t* const ret = (LZ4IO_prefs_t*)malloc(sizeof(*ret)); + if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); + ret->passThrough = 0; + ret->overwrite = 1; + ret->testMode = 0; + ret->blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; + ret->blockSize = 0; + ret->blockChecksum = 0; + ret->streamChecksum = 1; + ret->blockIndependence = 1; + ret->sparseFileSupport = 1; + ret->contentSizeFlag = 0; + ret->useDictionary = 0; + ret->favorDecSpeed = 0; + ret->dictionaryFilename = NULL; + ret->removeSrcFile = 0; + return ret; +} + +void LZ4IO_freePreferences(LZ4IO_prefs_t* prefs) +{ + free(prefs); +} + + +int LZ4IO_setDictionaryFilename(LZ4IO_prefs_t* const prefs, const char* dictionaryFilename) +{ + prefs->dictionaryFilename = dictionaryFilename; + prefs->useDictionary = dictionaryFilename != NULL; + return prefs->useDictionary; } +/* Default setting : passThrough = 0; return : passThrough mode (0/1) */ +int LZ4IO_setPassThrough(LZ4IO_prefs_t* const prefs, int yes) +{ + prefs->passThrough = (yes!=0); + return prefs->passThrough; +} + + /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ -int LZ4IO_setOverwrite(int yes) +int LZ4IO_setOverwrite(LZ4IO_prefs_t* const prefs, int yes) { - g_overwrite = (yes!=0); - return g_overwrite; + prefs->overwrite = (yes!=0); + return prefs->overwrite; } /* Default setting : testMode = 0; return : testMode (0/1) */ -int LZ4IO_setTestMode(int yes) +int LZ4IO_setTestMode(LZ4IO_prefs_t* const prefs, int yes) { - g_testMode = (yes!=0); - return g_testMode; + prefs->testMode = (yes!=0); + return prefs->testMode; } /* blockSizeID : valid values : 4-5-6-7 */ -size_t LZ4IO_setBlockSizeID(unsigned bsid) +size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned bsid) { static const size_t blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; static const unsigned minBlockSizeID = 4; static const unsigned maxBlockSizeID = 7; if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0; - g_blockSizeId = bsid; - return blockSizeTable[g_blockSizeId-minBlockSizeID]; + prefs->blockSizeId = (int)bsid; + prefs->blockSize = blockSizeTable[(unsigned)prefs->blockSizeId-minBlockSizeID]; + return prefs->blockSize; } -int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) +size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize) { - g_blockIndependence = (blockMode == LZ4IO_blockIndependent); - return g_blockIndependence; + static const size_t minBlockSize = 32; + static const size_t maxBlockSize = 4 MB; + unsigned bsid = 0; + if (blockSize < minBlockSize) blockSize = minBlockSize; + if (blockSize > maxBlockSize) blockSize = maxBlockSize; + prefs->blockSize = blockSize; + blockSize--; + /* find which of { 64k, 256k, 1MB, 4MB } is closest to blockSize */ + while (blockSize >>= 2) + bsid++; + if (bsid < 7) bsid = 7; + prefs->blockSizeId = (int)(bsid-3); + return prefs->blockSize; } -/* Default setting : no block checksum */ -int LZ4IO_setBlockChecksumMode(int enable) +/* Default setting : 1 == independent blocks */ +int LZ4IO_setBlockMode(LZ4IO_prefs_t* const prefs, LZ4IO_blockMode_t blockMode) { - g_blockChecksum = (enable != 0); - return g_blockChecksum; + prefs->blockIndependence = (blockMode == LZ4IO_blockIndependent); + return prefs->blockIndependence; } -/* Default setting : checksum enabled */ -int LZ4IO_setStreamChecksumMode(int enable) +/* Default setting : 0 == no block checksum */ +int LZ4IO_setBlockChecksumMode(LZ4IO_prefs_t* const prefs, int enable) { - g_streamChecksum = (enable != 0); - return g_streamChecksum; + prefs->blockChecksum = (enable != 0); + return prefs->blockChecksum; +} + +/* Default setting : 1 == checksum enabled */ +int LZ4IO_setStreamChecksumMode(LZ4IO_prefs_t* const prefs, int enable) +{ + prefs->streamChecksum = (enable != 0); + return prefs->streamChecksum; } /* Default setting : 0 (no notification) */ @@ -208,28 +270,30 @@ int LZ4IO_setNotificationLevel(int level) return g_displayLevel; } -/* Default setting : 0 (disabled) */ -int LZ4IO_setSparseFile(int enable) +/* Default setting : 1 (auto: enabled on file, disabled on stdout) */ +int LZ4IO_setSparseFile(LZ4IO_prefs_t* const prefs, int enable) { - g_sparseFileSupport = (enable!=0); - return g_sparseFileSupport; + prefs->sparseFileSupport = 2*(enable!=0); /* 2==force enable */ + return prefs->sparseFileSupport; } /* Default setting : 0 (disabled) */ -int LZ4IO_setContentSize(int enable) +int LZ4IO_setContentSize(LZ4IO_prefs_t* const prefs, int enable) { - g_contentSizeFlag = (enable!=0); - return g_contentSizeFlag; + prefs->contentSizeFlag = (enable!=0); + return prefs->contentSizeFlag; } /* Default setting : 0 (disabled) */ -void LZ4IO_favorDecSpeed(int favor) +void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor) { - g_favorDecSpeed = (favor!=0); + prefs->favorDecSpeed = (favor!=0); } -static U32 g_removeSrcFile = 0; -void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } +void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag) +{ + prefs->removeSrcFile = (flag>0); +} @@ -237,7 +301,6 @@ void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } ** ********************** LZ4 File / Pipe compression ********************* ** ** ************************************************************************ */ -static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; } @@ -263,25 +326,27 @@ static FILE* LZ4IO_openSrcFile(const char* srcFileName) } /** FIO_openDstFile() : - * condition : `dstFileName` must be non-NULL. + * prefs is writable, because sparseFileSupport might be updated. + * condition : `dstFileName` must be non-NULL. * @result : FILE* to `dstFileName`, or NULL if it fails */ -static FILE* LZ4IO_openDstFile(const char* dstFileName) +static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* const prefs) { FILE* f; + assert(dstFileName != NULL); if (!strcmp (dstFileName, stdoutmark)) { - DISPLAYLEVEL(4,"Using stdout for output\n"); + DISPLAYLEVEL(4, "Using stdout for output \n"); f = stdout; SET_BINARY_MODE(stdout); - if (g_sparseFileSupport==1) { - g_sparseFileSupport = 0; - DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); + if (prefs->sparseFileSupport==1) { + DISPLAYLEVEL(4, "Sparse File Support automatically disabled on stdout ;" + " to force-enable it, add --sparse command \n"); } } else { - if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ - f = fopen( dstFileName, "rb" ); - if (f != NULL) { /* dest exists, prompt for overwrite authorization */ - fclose(f); + if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ + FILE* const testf = fopen( dstFileName, "rb" ); + if (testf != NULL) { /* dest exists, prompt for overwrite authorization */ + fclose(testf); if (g_displayLevel <= 1) { /* No interaction possible */ DISPLAY("%s already exists; not overwritten \n", dstFileName); return NULL; @@ -299,7 +364,9 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName) } /* sparse file */ - if (f && g_sparseFileSupport) { SET_SPARSE_FILE_MODE(f); } + { int const sparseMode = (prefs->sparseFileSupport - (f==stdout)) > 0; + if (f && sparseMode) { SET_SPARSE_FILE_MODE(f); } + } return f; } @@ -329,57 +396,64 @@ static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSi /* LZ4IO_compressFilename_Legacy : * This function is intentionally "hidden" (not published in .h) * It generates compressed streams using the old 'legacy' format */ -int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) +int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, + int compressionlevel, const LZ4IO_prefs_t* prefs) { - int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); + typedef int (*compress_f)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); + compress_f const compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC; unsigned long long filesize = 0; unsigned long long compressedfilesize = MAGICNUMBER_SIZE; char* in_buff; char* out_buff; const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE); - FILE* finput; + FILE* const finput = LZ4IO_openSrcFile(input_filename); FILE* foutput; clock_t clockEnd; /* Init */ clock_t const clockStart = clock(); - compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC; + if (finput == NULL) + EXM_THROW(20, "%s : open file error ", input_filename); - finput = LZ4IO_openSrcFile(input_filename); - if (finput == NULL) EXM_THROW(20, "%s : open file error ", input_filename); - foutput = LZ4IO_openDstFile(output_filename); - if (foutput == NULL) { fclose(finput); EXM_THROW(20, "%s : open file error ", input_filename); } + foutput = LZ4IO_openDstFile(output_filename, prefs); + if (foutput == NULL) { + fclose(finput); + EXM_THROW(20, "%s : open file error ", input_filename); + } /* Allocate Memory */ in_buff = (char*)malloc(LEGACY_BLOCKSIZE); - out_buff = (char*)malloc(outBuffSize); - if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); + out_buff = (char*)malloc((size_t)outBuffSize + 4); + if (!in_buff || !out_buff) + EXM_THROW(21, "Allocation error : not enough memory"); /* Write Archive Header */ LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER); - { size_t const sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); } + if (fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) + EXM_THROW(22, "Write error : cannot write header"); /* Main Loop */ while (1) { - unsigned int outSize; + int outSize; /* Read Block */ - size_t const inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); + size_t const inSize = fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); if (inSize == 0) break; - if (inSize > LEGACY_BLOCKSIZE) EXM_THROW(23, "Read error : wrong fread() size report "); /* should be impossible */ + assert(inSize <= LEGACY_BLOCKSIZE); filesize += inSize; /* Compress Block */ outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel); - compressedfilesize += outSize+4; + assert(outSize >= 0); + compressedfilesize += (unsigned long long)outSize+4; DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ - LZ4IO_writeLE32(out_buff, outSize); - { size_t const sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); - if (sizeCheck!=(size_t)(outSize+4)) - EXM_THROW(24, "Write error : cannot write compressed block"); + assert(outSize > 0); + assert(outSize < outBuffSize); + LZ4IO_writeLE32(out_buff, (unsigned)outSize); + if (fwrite(out_buff, 1, (size_t)outSize+4, foutput) != (size_t)(outSize+4)) { + EXM_THROW(24, "Write error : cannot write compressed block"); } } if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename); @@ -399,11 +473,59 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output free(in_buff); free(out_buff); fclose(finput); - fclose(foutput); + if (strcmp(output_filename,stdoutmark)) fclose(foutput); /* do not close stdout */ return 0; } +#define FNSPACE 30 +/* LZ4IO_compressMultipleFilenames_Legacy : + * This function is intentionally "hidden" (not published in .h) + * It generates multiple compressed streams using the old 'legacy' format */ +int LZ4IO_compressMultipleFilenames_Legacy( + const char** inFileNamesTable, int ifntSize, + const char* suffix, + int compressionLevel, const LZ4IO_prefs_t* prefs) +{ + int i; + int missed_files = 0; + char* dstFileName = (char*)malloc(FNSPACE); + size_t ofnSize = FNSPACE; + const size_t suffixSize = strlen(suffix); + + if (dstFileName == NULL) return ifntSize; /* not enough memory */ + + /* loop on each file */ + for (i=0; i<ifntSize; i++) { + size_t const ifnSize = strlen(inFileNamesTable[i]); + if (!strcmp(suffix, stdoutmark)) { + missed_files += LZ4IO_compressFilename_Legacy( + inFileNamesTable[i], stdoutmark, + compressionLevel, prefs); + continue; + } + + if (ofnSize <= ifnSize+suffixSize+1) { + free(dstFileName); + ofnSize = ifnSize + 20; + dstFileName = (char*)malloc(ofnSize); + if (dstFileName==NULL) { + return ifntSize; + } } + strcpy(dstFileName, inFileNamesTable[i]); + strcat(dstFileName, suffix); + + missed_files += LZ4IO_compressFilename_Legacy( + inFileNamesTable[i], dstFileName, + compressionLevel, prefs); + } + + /* Close & Free */ + free(dstFileName); + + return missed_files; +} + /********************************************* * Compression using Frame format @@ -418,21 +540,20 @@ typedef struct { LZ4F_CDict* cdict; } cRess_t; -static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { +static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename) +{ size_t readSize; size_t dictEnd = 0; size_t dictLen = 0; size_t dictStart; size_t circularBufSize = LZ4_MAX_DICT_SIZE; - char* circularBuf; - char* dictBuf; + char* circularBuf = (char*)malloc(circularBufSize); + char* dictBuf; FILE* dictFile; + if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory for circular buffer"); if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided"); - circularBuf = (char *) malloc(circularBufSize); - if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory"); - dictFile = LZ4IO_openSrcFile(dictFilename); if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file"); @@ -462,7 +583,7 @@ static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { circularBuf = NULL; } else { /* Otherwise, we will alloc a new buffer and copy our dict into that. */ - dictBuf = (char *) malloc(dictLen ? dictLen : 1); + dictBuf = (char *)malloc(dictLen ? dictLen : 1); if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory"); memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart); @@ -475,23 +596,22 @@ static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { return dictBuf; } -static LZ4F_CDict* LZ4IO_createCDict(void) { +static LZ4F_CDict* LZ4IO_createCDict(const LZ4IO_prefs_t* const prefs) +{ size_t dictionarySize; void* dictionaryBuffer; LZ4F_CDict* cdict; - if (!g_useDictionary) { - return NULL; - } - dictionaryBuffer = LZ4IO_createDict(g_dictionaryFilename, &dictionarySize); + if (!prefs->useDictionary) return NULL; + dictionaryBuffer = LZ4IO_createDict(&dictionarySize, prefs->dictionaryFilename); if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize); free(dictionaryBuffer); return cdict; } -static cRess_t LZ4IO_createCResources(void) +static cRess_t LZ4IO_createCResources(const LZ4IO_prefs_t* const prefs) { - const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); + const size_t blockSize = prefs->blockSize; cRess_t ress; LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION); @@ -504,7 +624,7 @@ static cRess_t LZ4IO_createCResources(void) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); - ress.cdict = LZ4IO_createCDict(); + ress.cdict = LZ4IO_createCDict(prefs); return ress; } @@ -526,38 +646,39 @@ static void LZ4IO_freeCResources(cRess_t ress) * result : 0 : compression completed correctly * 1 : missing or pb opening srcFileName */ -static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) +static int +LZ4IO_compressFilename_extRess(cRess_t ress, + const char* srcFileName, const char* dstFileName, + int compressionLevel, const LZ4IO_prefs_t* const io_prefs) { unsigned long long filesize = 0; unsigned long long compressedfilesize = 0; - FILE* srcFile; FILE* dstFile; void* const srcBuffer = ress.srcBuffer; void* const dstBuffer = ress.dstBuffer; const size_t dstBufferSize = ress.dstBufferSize; - const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); + const size_t blockSize = io_prefs->blockSize; size_t readSize; LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */ LZ4F_preferences_t prefs; /* Init */ - srcFile = LZ4IO_openSrcFile(srcFileName); + FILE* const srcFile = LZ4IO_openSrcFile(srcFileName); if (srcFile == NULL) return 1; - dstFile = LZ4IO_openDstFile(dstFileName); + dstFile = LZ4IO_openDstFile(dstFileName, io_prefs); if (dstFile == NULL) { fclose(srcFile); return 1; } memset(&prefs, 0, sizeof(prefs)); - /* Set compression parameters */ prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence; - prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId; - prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum; - prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum; - prefs.favorDecSpeed = g_favorDecSpeed; - if (g_contentSizeFlag) { - U64 const fileSize = UTIL_getFileSize(srcFileName); + prefs.frameInfo.blockMode = (LZ4F_blockMode_t)io_prefs->blockIndependence; + prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)io_prefs->blockSizeId; + prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)io_prefs->blockChecksum; + prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)io_prefs->streamChecksum; + prefs.favorDecSpeed = io_prefs->favorDecSpeed; + if (io_prefs->contentSizeFlag) { + U64 const fileSize = UTIL_getOpenFileSize(srcFile); prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ if (fileSize==0) DISPLAYLEVEL(3, "Warning : cannot determine input content size \n"); @@ -571,41 +692,41 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* single-block file */ if (readSize < blockSize) { /* Compress in single pass */ - size_t cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs); - if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); + size_t const cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs); + if (LZ4F_isError(cSize)) + EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); compressedfilesize = cSize; DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ /* Write Block */ - { size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); - if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block"); + if (fwrite(dstBuffer, 1, cSize, dstFile) != cSize) { + EXM_THROW(32, "Write error : failed writing single-block compressed frame"); } } else /* multiple-blocks file */ { - /* Write Archive Header */ - size_t headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs); + /* Write Frame Header */ + size_t const headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs); if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); - { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } + if (fwrite(dstBuffer, 1, headerSize, dstFile) != headerSize) + EXM_THROW(34, "Write error : cannot write header"); compressedfilesize += headerSize; - /* Main Loop */ + /* Main Loop - one block at a time */ while (readSize>0) { - size_t outSize; - - /* Compress Block */ - outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); - if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize)); + size_t const outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); + if (LZ4F_isError(outSize)) + EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize)); compressedfilesize += outSize; - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", + (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ - { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); - if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } + if (fwrite(dstBuffer, 1, outSize, dstFile) != outSize) + EXM_THROW(36, "Write error : cannot write compressed block"); /* Read next block */ readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); @@ -613,18 +734,18 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, } if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); - /* End of Stream mark */ - headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); - if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); - - { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } - compressedfilesize += headerSize; - } + /* End of Frame mark */ + { size_t const endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); + if (LZ4F_isError(endSize)) + EXM_THROW(38, "End of frame error : %s", LZ4F_getErrorName(endSize)); + if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize) + EXM_THROW(39, "Write error : cannot write end of frame"); + compressedfilesize += endSize; + } } - /* Release files */ + /* Release file handlers */ fclose (srcFile); - fclose (dstFile); + if (strcmp(dstFileName,stdoutmark)) fclose (dstFile); /* do not close stdout */ /* Copy owner, file permissions and modification time */ { stat_t statbuf; @@ -635,7 +756,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, UTIL_setFileStat(dstFileName, &statbuf); } } - if (g_removeSrcFile) { /* remove source file : --rm */ + if (io_prefs->removeSrcFile) { /* remove source file : --rm */ if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } @@ -650,13 +771,13 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, } -int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) +int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel, const LZ4IO_prefs_t* prefs) { UTIL_time_t const timeStart = UTIL_getTime(); clock_t const cpuStart = clock(); - cRess_t const ress = LZ4IO_createCResources(); + cRess_t const ress = LZ4IO_createCResources(prefs); - int const result = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); + int const result = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel, prefs); /* Free resources */ LZ4IO_freeCResources(ress); @@ -674,8 +795,11 @@ int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int } -#define FNSPACE 30 -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) +int LZ4IO_compressMultipleFilenames( + const char** inFileNamesTable, int ifntSize, + const char* suffix, + int compressionLevel, + const LZ4IO_prefs_t* prefs) { int i; int missed_files = 0; @@ -685,16 +809,31 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, cRess_t ress; if (dstFileName == NULL) return ifntSize; /* not enough memory */ - ress = LZ4IO_createCResources(); + ress = LZ4IO_createCResources(prefs); /* loop on each file */ for (i=0; i<ifntSize; i++) { size_t const ifnSize = strlen(inFileNamesTable[i]); - if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); if (dstFileName==NULL) { LZ4IO_freeCResources(ress); return ifntSize; } } + if (!strcmp(suffix, stdoutmark)) { + missed_files += LZ4IO_compressFilename_extRess(ress, + inFileNamesTable[i], stdoutmark, + compressionLevel, prefs); + continue; + } + if (ofnSize <= ifnSize+suffixSize+1) { + free(dstFileName); + ofnSize = ifnSize + 20; + dstFileName = (char*)malloc(ofnSize); + if (dstFileName==NULL) { + LZ4IO_freeCResources(ress); + return ifntSize; + } } strcpy(dstFileName, inFileNamesTable[i]); strcat(dstFileName, suffix); - missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel); + missed_files += LZ4IO_compressFilename_extRess(ress, + inFileNamesTable[i], dstFileName, + compressionLevel, prefs); } /* Close & Free */ @@ -709,18 +848,23 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, /* ********************** LZ4 file-stream Decompression **************** */ /* ********************************************************************* */ +/* It's presumed that s points to a memory space of size >= 4 */ static unsigned LZ4IO_readLE32 (const void* s) { const unsigned char* const srcPtr = (const unsigned char*)s; unsigned value32 = srcPtr[0]; - value32 += (srcPtr[1]<<8); - value32 += (srcPtr[2]<<16); - value32 += ((unsigned)srcPtr[3])<<24; + value32 += (unsigned)srcPtr[1] << 8; + value32 += (unsigned)srcPtr[2] << 16; + value32 += (unsigned)srcPtr[3] << 24; return value32; } -static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) +static unsigned +LZ4IO_fwriteSparse(FILE* file, + const void* buffer, size_t bufferSize, + int sparseFileSupport, + unsigned storedSkips) { const size_t sizeT = sizeof(size_t); const size_t maskT = sizeT -1 ; @@ -729,8 +873,9 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer size_t bufferSizeT = bufferSize / sizeT; const size_t* const bufferTEnd = bufferT + bufferSizeT; const size_t segmentSizeT = (32 KB) / sizeT; + int const sparseMode = (sparseFileSupport - (file==stdout)) > 0; - if (!g_sparseFileSupport) { /* normal write */ + if (!sparseMode) { /* normal write */ size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); return 0; @@ -778,7 +923,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); storedSkips = 0; - { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); + { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file); if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); } } } @@ -788,24 +933,24 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) { - if (storedSkips>0) { /* implies g_sparseFileSupport>0 */ - int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR); - if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); - { const char lastZeroByte[1] = { 0 }; - size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); - if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); - } } + if (storedSkips>0) { /* implies sparseFileSupport>0 */ + const char lastZeroByte[1] = { 0 }; + if (UTIL_fseek(file, storedSkips-1, SEEK_CUR) != 0) + EXM_THROW(69, "Final skip error (sparse file)\n"); + if (fwrite(lastZeroByte, 1, 1, file) != 1) + EXM_THROW(69, "Write error : cannot write last zero\n"); + } } static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */ -static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) +static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs) { unsigned long long streamSize = 0; unsigned storedSkips = 0; /* Allocate Memory */ - char* const in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + char* const in_buff = (char*)malloc((size_t)LZ4_compressBound(LEGACY_BLOCKSIZE)); char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE); if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); @@ -829,11 +974,11 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); } /* Decode Block */ - { int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); + { int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, (int)blockSize, LEGACY_BLOCKSIZE); if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); - streamSize += decodeSize; + streamSize += (unsigned long long)decodeSize; /* Write Block */ - storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); /* success or die */ + storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, (size_t)decodeSize, prefs->sparseFileSupport, storedSkips); /* success or die */ } } if (ferror(finput)) EXM_THROW(54, "Read error : ferror"); @@ -859,19 +1004,20 @@ typedef struct { size_t dictBufferSize; } dRess_t; -static void LZ4IO_loadDDict(dRess_t* ress) { - if (!g_useDictionary) { +static void LZ4IO_loadDDict(dRess_t* ress, const LZ4IO_prefs_t* const prefs) +{ + if (!prefs->useDictionary) { ress->dictBuffer = NULL; ress->dictBufferSize = 0; return; } - ress->dictBuffer = LZ4IO_createDict(g_dictionaryFilename, &ress->dictBufferSize); + ress->dictBuffer = LZ4IO_createDict(&ress->dictBufferSize, prefs->dictionaryFilename); if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); } static const size_t LZ4IO_dBufferSize = 64 KB; -static dRess_t LZ4IO_createDResources(void) +static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs) { dRess_t ress; @@ -886,7 +1032,7 @@ static dRess_t LZ4IO_createDResources(void) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); - LZ4IO_loadDDict(&ress); + LZ4IO_loadDDict(&ress, prefs); ress.dstFile = NULL; return ress; @@ -902,7 +1048,10 @@ static void LZ4IO_freeDResources(dRess_t ress) } -static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile) +static unsigned long long +LZ4IO_decompressLZ4F(dRess_t ress, + FILE* const srcFile, FILE* const dstFile, + const LZ4IO_prefs_t* const prefs) { unsigned long long filesize = 0; LZ4F_errorCode_t nextToLoad; @@ -937,8 +1086,8 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE /* Write Block */ if (decodedBytes) { - if (!g_testMode) - storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); + if (!prefs->testMode) + storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, prefs->sparseFileSupport, storedSkips); filesize += decodedBytes; DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); } @@ -949,29 +1098,37 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE /* can be out because readSize == 0, which could be an fread() error */ if (ferror(srcFile)) EXM_THROW(67, "Read error"); - if (!g_testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips); + if (!prefs->testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips); if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream"); return filesize; } +/* LZ4IO_passThrough: + * just output the same content as input, no decoding. + * This is a capability of zcat, and by extension lz4cat + * MNstore : contain the first MAGICNUMBER_SIZE bytes already read from finput + */ #define PTSIZE (64 KB) #define PTSIZET (PTSIZE / sizeof(size_t)) -static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) +static unsigned long long +LZ4IO_passThrough(FILE* finput, FILE* foutput, + unsigned char MNstore[MAGICNUMBER_SIZE], + int sparseFileSupport) { size_t buffer[PTSIZET]; size_t readBytes = 1; unsigned long long total = MAGICNUMBER_SIZE; unsigned storedSkips = 0; - size_t const sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error"); - + if (fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) { + EXM_THROW(50, "Pass-through write error"); + } while (readBytes) { - readBytes = fread(buffer, 1, PTSIZE, finput); + readBytes = fread(buffer, 1, sizeof(buffer), finput); total += readBytes; - storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, storedSkips); + storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, sparseFileSupport, storedSkips); } if (ferror(finput)) EXM_THROW(51, "Read Error"); @@ -998,7 +1155,10 @@ static int fseek_u32(FILE *fp, unsigned offset, int where) } #define ENDOFSTREAM ((unsigned long long)-1) -static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) +static unsigned long long +selectDecoder(dRess_t ress, + FILE* finput, FILE* foutput, + const LZ4IO_prefs_t* const prefs) { unsigned char MNstore[MAGICNUMBER_SIZE]; unsigned magicNumber; @@ -1024,10 +1184,10 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu switch(magicNumber) { case LZ4IO_MAGICNUMBER: - return LZ4IO_decompressLZ4F(ress, finput, foutput); + return LZ4IO_decompressLZ4F(ress, finput, foutput, prefs); case LEGACY_MAGICNUMBER: DISPLAYLEVEL(4, "Detected : Legacy format \n"); - return LZ4IO_decodeLegacyStream(finput, foutput); + return LZ4IO_decodeLegacyStream(finput, foutput, prefs); case LZ4IO_SKIPPABLE0: DISPLAYLEVEL(4, "Skipping detected skippable area \n"); { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); @@ -1044,9 +1204,9 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu default: if (nbFrames == 1) { /* just started */ /* Wrong magic number at the beginning of 1st stream */ - if (!g_testMode && g_overwrite) { + if (!prefs->testMode && prefs->overwrite && prefs->passThrough) { nbFrames = 0; - return LZ4IO_passThrough(finput, foutput, MNstore); + return LZ4IO_passThrough(finput, foutput, MNstore, prefs->sparseFileSupport); } EXM_THROW(44,"Unrecognized header : file cannot be decoded"); } @@ -1061,7 +1221,10 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu } -static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename) +static int +LZ4IO_decompressSrcFile(dRess_t ress, + const char* input_filename, const char* output_filename, + const LZ4IO_prefs_t* const prefs) { FILE* const foutput = ress.dstFile; unsigned long long filesize = 0; @@ -1069,18 +1232,19 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con /* Init */ FILE* const finput = LZ4IO_openSrcFile(input_filename); if (finput==NULL) return 1; + assert(foutput != NULL); /* Loop over multiple streams */ for ( ; ; ) { /* endless loop, see break condition */ unsigned long long const decodedSize = - selectDecoder(ress, finput, foutput); + selectDecoder(ress, finput, foutput, prefs); if (decodedSize == ENDOFSTREAM) break; filesize += decodedSize; } /* Close input */ fclose(finput); - if (g_removeSrcFile) { /* --rm */ + if (prefs->removeSrcFile) { /* --rm */ if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); } @@ -1094,11 +1258,14 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con } -static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename) +static int +LZ4IO_decompressDstFile(dRess_t ress, + const char* input_filename, const char* output_filename, + const LZ4IO_prefs_t* const prefs) { stat_t statbuf; int stat_result = 0; - FILE* const foutput = LZ4IO_openDstFile(output_filename); + FILE* const foutput = LZ4IO_openDstFile(output_filename, prefs); if (foutput==NULL) return 1; /* failure */ if ( strcmp(input_filename, stdinmark) @@ -1106,7 +1273,7 @@ static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, con stat_result = 1; ress.dstFile = foutput; - LZ4IO_decompressSrcFile(ress, input_filename, output_filename); + LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs); fclose(foutput); @@ -1122,12 +1289,12 @@ static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, con } -int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) +int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename, const LZ4IO_prefs_t* prefs) { - dRess_t const ress = LZ4IO_createDResources(); + dRess_t const ress = LZ4IO_createDResources(prefs); clock_t const start = clock(); - int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename); + int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename, prefs); clock_t const end = clock(); double const seconds = (double)(end - start) / CLOCKS_PER_SEC; @@ -1138,7 +1305,10 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file } -int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) +int LZ4IO_decompressMultipleFilenames( + const char** inFileNamesTable, int ifntSize, + const char* suffix, + const LZ4IO_prefs_t* prefs) { int i; int skippedFiles = 0; @@ -1146,19 +1316,24 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz char* outFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; size_t const suffixSize = strlen(suffix); - dRess_t ress = LZ4IO_createDResources(); + dRess_t ress = LZ4IO_createDResources(prefs); - if (outFileName==NULL) return ifntSize; /* not enough memory */ - ress.dstFile = LZ4IO_openDstFile(stdoutmark); + if (outFileName==NULL) EXM_THROW(70, "Memory allocation error"); + ress.dstFile = LZ4IO_openDstFile(stdoutmark, prefs); for (i=0; i<ifntSize; i++) { size_t const ifnSize = strlen(inFileNamesTable[i]); const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize; if (!strcmp(suffix, stdoutmark)) { - missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark); + missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark, prefs); continue; } - if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) return ifntSize; } + if (ofnSize <= ifnSize-suffixSize+1) { + free(outFileName); + ofnSize = ifnSize + 20; + outFileName = (char*)malloc(ofnSize); + if (outFileName==NULL) EXM_THROW(71, "Memory allocation error"); + } if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) { DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); skippedFiles++; @@ -1166,10 +1341,337 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz } memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize); outFileName[ifnSize-suffixSize] = '\0'; - missingFiles += LZ4IO_decompressDstFile(ress, inFileNamesTable[i], outFileName); + missingFiles += LZ4IO_decompressDstFile(ress, inFileNamesTable[i], outFileName, prefs); } LZ4IO_freeDResources(ress); free(outFileName); return missingFiles + skippedFiles; } + + +/* ********************************************************************* */ +/* ********************** LZ4 --list command *********************** */ +/* ********************************************************************* */ + +typedef enum +{ + lz4Frame = 0, + legacyFrame, + skippableFrame +} LZ4IO_frameType_t; + +typedef struct { + LZ4F_frameInfo_t lz4FrameInfo; + LZ4IO_frameType_t frameType; +} LZ4IO_frameInfo_t; + +#define LZ4IO_INIT_FRAMEINFO { LZ4F_INIT_FRAMEINFO, lz4Frame } + +typedef struct { + const char* fileName; + unsigned long long fileSize; + unsigned long long frameCount; + LZ4IO_frameInfo_t frameSummary; + unsigned short eqFrameTypes; + unsigned short eqBlockTypes; + unsigned short allContentSize; +} LZ4IO_cFileInfo_t; + +#define LZ4IO_INIT_CFILEINFO { NULL, 0ULL, 0, LZ4IO_INIT_FRAMEINFO, 1, 1, 1 } + +typedef enum { LZ4IO_LZ4F_OK, LZ4IO_format_not_known, LZ4IO_not_a_file } LZ4IO_infoResult; + +static const char * LZ4IO_frameTypeNames[] = {"LZ4Frame", "LegacyFrame", "SkippableFrame" }; + +/* Read block headers and skip block data + Return total blocks size for this frame including block headers, + block checksums and content checksums. + returns 0 in case it can't succesfully skip block data. + Assumes SEEK_CUR after frame header. + */ +static unsigned long long +LZ4IO_skipBlocksData(FILE* finput, + const LZ4F_blockChecksum_t blockChecksumFlag, + const LZ4F_contentChecksum_t contentChecksumFlag) +{ + unsigned char blockInfo[LZ4F_BLOCK_HEADER_SIZE]; + unsigned long long totalBlocksSize = 0; + for (;;) { + if (!fread(blockInfo, 1, LZ4F_BLOCK_HEADER_SIZE, finput)) { + if (feof(finput)) return totalBlocksSize; + return 0; + } + totalBlocksSize += LZ4F_BLOCK_HEADER_SIZE; + { const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU; + const unsigned long nextBlock = nextCBlockSize + (blockChecksumFlag * LZ4F_BLOCK_CHECKSUM_SIZE); + if (nextCBlockSize == 0) { + /* Reached EndMark */ + if (contentChecksumFlag) { + /* Skip content checksum */ + if (UTIL_fseek(finput, LZ4F_CONTENT_CHECKSUM_SIZE, SEEK_CUR) != 0) { + return 0; + } + totalBlocksSize += LZ4F_CONTENT_CHECKSUM_SIZE; + } + break; + } + totalBlocksSize += nextBlock; + /* skip to the next block */ + assert(nextBlock < LONG_MAX); + if (UTIL_fseek(finput, (long)nextBlock, SEEK_CUR) != 0) return 0; + } } + return totalBlocksSize; +} + +/* For legacy frames only. + Read block headers and skip block data. + Return total blocks size for this frame including block headers. + or 0 in case it can't succesfully skip block data. + This works as long as legacy block header size = magic number size. + Assumes SEEK_CUR after frame header. + */ +static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) +{ + unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE]; + unsigned long long totalBlocksSize = 0; + LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE); + for (;;) { + if (!fread(blockInfo, 1, LZIO_LEGACY_BLOCK_HEADER_SIZE, finput)) { + if (feof(finput)) return totalBlocksSize; + return 0; + } + { const unsigned int nextCBlockSize = LZ4IO_readLE32(&blockInfo); + if ( nextCBlockSize == LEGACY_MAGICNUMBER || + nextCBlockSize == LZ4IO_MAGICNUMBER || + LZ4IO_isSkippableMagicNumber(nextCBlockSize)) { + /* Rewind back. we want cursor at the begining of next frame.*/ + if (fseek(finput, -LZIO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) { + return 0; + } + break; + } + totalBlocksSize += LZIO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize; + /* skip to the next block */ + if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) { + return 0; + } } } + return totalBlocksSize; +} + +/* LZ4IO_blockTypeID: + * return human-readable block type, following command line convention + * buffer : must be a valid memory area of at least 4 bytes */ +const char* LZ4IO_blockTypeID(LZ4F_blockSizeID_t sizeID, LZ4F_blockMode_t blockMode, char buffer[4]) +{ + buffer[0] = 'B'; + assert(sizeID >= 4); assert(sizeID <= 7); + buffer[1] = (char)(sizeID + '0'); + buffer[2] = (blockMode == LZ4F_blockIndependent) ? 'I' : 'D'; + buffer[3] = 0; + return buffer; +} + +/* buffer : must be valid memory area of at least 10 bytes */ +static const char* LZ4IO_toHuman(long double size, char *buf) +{ + const char units[] = {"\0KMGTPEZY"}; + size_t i = 0; + for (; size >= 1024; i++) size /= 1024; + sprintf(buf, "%.2Lf%c", size, units[i]); + return buf; +} + +/* Get filename without path prefix */ +static const char* LZ4IO_baseName(const char* input_filename) +{ + const char* b = strrchr(input_filename, '/'); + if (!b) b = strrchr(input_filename, '\\'); + if (!b) return input_filename; + return b + 1; +} + +/* Report frame/s information (--list) in verbose mode (-v). + * Will populate file info with fileName and frameSummary where applicable. + * - TODO : + * + report nb of blocks, hence max. possible decompressed size (when not reported in header) + */ +static LZ4IO_infoResult +LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename) +{ + LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */ + unsigned char buffer[LZ4F_HEADER_SIZE_MAX]; + FILE* const finput = LZ4IO_openSrcFile(input_filename); + + if (finput == NULL) return LZ4IO_not_a_file; + cfinfo->fileSize = UTIL_getOpenFileSize(finput); + + while (!feof(finput)) { + LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO; + unsigned magicNumber; + /* Get MagicNumber */ + { size_t const nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput); + if (nbReadBytes == 0) { break; } /* EOF */ + result = LZ4IO_format_not_known; /* default result (error) */ + if (nbReadBytes != MAGICNUMBER_SIZE) { + EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + } } + magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */ + if (LZ4IO_isSkippableMagicNumber(magicNumber)) + magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ + + switch (magicNumber) { + case LZ4IO_MAGICNUMBER: + if (cfinfo->frameSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0; + /* Get frame info */ + { const size_t readBytes = fread(buffer + MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN - MAGICNUMBER_SIZE, finput); + if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename); + } + { size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN); + if (LZ4F_isError(hSize)) break; + if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) { + /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/ + const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput); + if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename); + } + /* Create decompression context */ + { LZ4F_dctx* dctx; + if ( LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) break; + { unsigned const frameInfoError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize)); + LZ4F_freeDecompressionContext(dctx); + if (frameInfoError) break; + if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID || + cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode) + && cfinfo->frameCount != 0) + cfinfo->eqBlockTypes = 0; + { const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput, + frameInfo.lz4FrameInfo.blockChecksumFlag, + frameInfo.lz4FrameInfo.contentChecksumFlag); + if (totalBlocksSize) { + char bTypeBuffer[5]; + LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer); + DISPLAYLEVEL(3, " %6llu %14s %5s %8s", + cfinfo->frameCount + 1, + LZ4IO_frameTypeNames[frameInfo.frameType], + bTypeBuffer, + frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-"); + if (frameInfo.lz4FrameInfo.contentSize) { + { double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100; + DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n", + totalBlocksSize + hSize, + frameInfo.lz4FrameInfo.contentSize, + ratio); + } + /* Now we've consumed frameInfo we can use it to store the total contentSize */ + frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize; + } + else { + DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-"); + cfinfo->allContentSize = 0; + } + result = LZ4IO_LZ4F_OK; + } } } } } + break; + case LEGACY_MAGICNUMBER: + frameInfo.frameType = legacyFrame; + if (cfinfo->frameSummary.frameType != legacyFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0; + cfinfo->eqBlockTypes = 0; + cfinfo->allContentSize = 0; + { const unsigned long long totalBlocksSize = LZ4IO_skipLegacyBlocksData(finput); + if (totalBlocksSize) { + DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20llu %20s %9s\n", + cfinfo->frameCount + 1, + LZ4IO_frameTypeNames[frameInfo.frameType], + "-", "-", + totalBlocksSize + 4, + "-", "-"); + result = LZ4IO_LZ4F_OK; + } } + break; + case LZ4IO_SKIPPABLE0: + frameInfo.frameType = skippableFrame; + if (cfinfo->frameSummary.frameType != skippableFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0; + cfinfo->eqBlockTypes = 0; + cfinfo->allContentSize = 0; + { size_t const nbReadBytes = fread(buffer, 1, 4, finput); + if (nbReadBytes != 4) + EXM_THROW(42, "Stream error : skippable size unreadable"); + } + { unsigned const size = LZ4IO_readLE32(buffer); + int const errorNb = fseek_u32(finput, size, SEEK_CUR); + if (errorNb != 0) + EXM_THROW(43, "Stream error : cannot skip skippable area"); + DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n", + cfinfo->frameCount + 1, + "SkippableFrame", + "-", "-", size + 8, "-", "-"); + + result = LZ4IO_LZ4F_OK; + } + break; + default: + { long int const position = ftell(finput); /* only works for files < 2 GB */ + DISPLAYLEVEL(3, "Stream followed by undecodable data "); + if (position != -1L) + DISPLAYLEVEL(3, "at position %i ", (int)position); + DISPLAYLEVEL(3, "\n"); + } + break; + } + if (result != LZ4IO_LZ4F_OK) break; + cfinfo->frameSummary = frameInfo; + cfinfo->frameCount++; + } /* while (!feof(finput)) */ + fclose(finput); + return result; +} + + +int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx) +{ + int result = 0; + size_t idx = 0; + if (g_displayLevel < 3) { + DISPLAYOUT("%10s %14s %5s %11s %13s %9s %s\n", + "Frames", "Type", "Block", "Compressed", "Uncompressed", "Ratio", "Filename"); + } + for (; idx < ifnIdx; idx++) { + /* Get file info */ + LZ4IO_cFileInfo_t cfinfo = LZ4IO_INIT_CFILEINFO; + cfinfo.fileName = LZ4IO_baseName(inFileNames[idx]); + if (!UTIL_isRegFile(inFileNames[idx])) { + DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]); + return 0; + } + DISPLAYLEVEL(3, "%s(%llu/%llu)\n", cfinfo.fileName, (unsigned long long)idx + 1, (unsigned long long)ifnIdx); + DISPLAYLEVEL(3, " %6s %14s %5s %8s %20s %20s %9s\n", + "Frame", "Type", "Block", "Checksum", "Compressed", "Uncompressed", "Ratio") + { LZ4IO_infoResult const op_result = LZ4IO_getCompressedFileInfo(&cfinfo, inFileNames[idx]); + if (op_result != LZ4IO_LZ4F_OK) { + assert(op_result == LZ4IO_format_not_known); + DISPLAYLEVEL(1, "lz4: %s: File format not recognized \n", inFileNames[idx]); + return 0; + } } + DISPLAYLEVEL(3, "\n"); + if (g_displayLevel < 3) { + /* Display Summary */ + { char buffers[3][10]; + DISPLAYOUT("%10llu %14s %5s %11s %13s ", + cfinfo.frameCount, + cfinfo.eqFrameTypes ? LZ4IO_frameTypeNames[cfinfo.frameSummary.frameType] : "-" , + cfinfo.eqBlockTypes ? LZ4IO_blockTypeID(cfinfo.frameSummary.lz4FrameInfo.blockSizeID, + cfinfo.frameSummary.lz4FrameInfo.blockMode, buffers[0]) : "-", + LZ4IO_toHuman((long double)cfinfo.fileSize, buffers[1]), + cfinfo.allContentSize ? LZ4IO_toHuman((long double)cfinfo.frameSummary.lz4FrameInfo.contentSize, buffers[2]) : "-"); + if (cfinfo.allContentSize) { + double const ratio = (double)cfinfo.fileSize / cfinfo.frameSummary.lz4FrameInfo.contentSize * 100; + DISPLAYOUT("%9.2f%% %s \n", ratio, cfinfo.fileName); + } else { + DISPLAYOUT("%9s %s\n", + "-", + cfinfo.fileName); + } } } /* if (g_displayLevel < 3) */ + } /* for (; idx < ifnIdx; idx++) */ + + return result; +} |