summaryrefslogtreecommitdiff
path: root/programs
diff options
context:
space:
mode:
Diffstat (limited to 'programs')
-rw-r--r--programs/.gitignore1
-rw-r--r--programs/Makefile93
-rw-r--r--programs/README.md27
-rw-r--r--programs/bench.c284
-rw-r--r--programs/bench.h3
-rw-r--r--programs/lz4-exe.rc.in27
-rw-r--r--programs/lz4.144
-rw-r--r--programs/lz4.1.md48
-rw-r--r--programs/lz4cli.c235
-rw-r--r--programs/lz4io.c898
-rw-r--r--programs/lz4io.h64
-rw-r--r--programs/platform.h11
-rw-r--r--programs/util.h161
13 files changed, 1421 insertions, 475 deletions
diff --git a/programs/.gitignore b/programs/.gitignore
index daa7f14..9ffadd9 100644
--- a/programs/.gitignore
+++ b/programs/.gitignore
@@ -4,6 +4,7 @@ unlz4
lz4cat
lz4c
lz4c32
+lz4-wlib
datagen
frametest
frametest32
diff --git a/programs/Makefile b/programs/Makefile
index bd33d9b..c1053f6 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -22,7 +22,7 @@
#
# You can contact the author at :
# - LZ4 homepage : http://www.lz4.org
-# - LZ4 source repository : https://github.com/Cyan4973/lz4
+# - LZ4 source repository : https://github.com/lz4/lz4
# ##########################################################################
# lz4 : Command Line Utility, supporting gzip-like arguments
# lz4c : CLU, supporting also legacy lz4demo arguments
@@ -41,12 +41,13 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
LIBVER := $(shell echo $(LIBVER_SCRIPT))
-SRCFILES := $(sort $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c))
-OBJFILES := $(SRCFILES:.c=.o)
+LIBFILES = $(wildcard $(LZ4DIR)/*.c)
+SRCFILES = $(sort $(LIBFILES) $(wildcard *.c))
+OBJFILES = $(SRCFILES:.c=.o)
CPPFLAGS += -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
-DEBUGFLAGS:=-Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
+DEBUGFLAGS= -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
@@ -56,17 +57,7 @@ LZ4_VERSION=$(LIBVER)
MD2ROFF = ronn
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)"
-
-# Define *.exe as extension for Windows systems
-ifneq (,$(filter Windows%,$(OS)))
-EXT :=.exe
-VOID := nul
-else
-EXT :=
-VOID := /dev/null
-endif
-
-
+include ../Makefile.inc
default: lz4-release
@@ -75,14 +66,44 @@ all: lz4 lz4c
all32: CFLAGS+=-m32
all32: all
-lz4: $(OBJFILES)
+ifeq ($(WINBASED),yes)
+lz4-exe.rc: lz4-exe.rc.in
+ @echo creating executable resource
+ $(Q)sed -e 's|@PROGNAME@|lz4|' \
+ -e 's|@LIBVER_MAJOR@|$(LIBVER_MAJOR)|g' \
+ -e 's|@LIBVER_MINOR@|$(LIBVER_MINOR)|g' \
+ -e 's|@LIBVER_PATCH@|$(LIBVER_PATCH)|g' \
+ -e 's|@EXT@|$(EXT)|g' \
+ $< >$@
+
+lz4-exe.o: lz4-exe.rc
+ $(WINDRES) -i lz4-exe.rc -o lz4-exe.o
+
+lz4: $(OBJFILES) lz4-exe.o
$(CC) $(FLAGS) $^ -o $@$(EXT)
+else
+lz4: $(OBJFILES)
+ $(CC) $(FLAGS) $(OBJFILES) -o $@$(EXT) $(LDLIBS)
+endif
+.PHONY: lz4-release
lz4-release: DEBUGFLAGS=
lz4-release: lz4
+lz4-wlib: LIBFILES =
+lz4-wlib: SRCFILES+= $(LZ4DIR)/xxhash.c # benchmark unit needs XXH64()
+lz4-wlib: LDFLAGS += -L $(LZ4DIR)
+lz4-wlib: LDLIBS = -llz4
+lz4-wlib: liblz4 $(OBJFILES)
+ @echo WARNING: $@ must link to an extended variant of the dynamic library which also exposes unstable symbols
+ $(CC) $(FLAGS) $(OBJFILES) -o $@$(EXT) $(LDLIBS)
+
+.PHONY:liblz4
+liblz4:
+ CPPFLAGS="-DLZ4F_PUBLISH_STATIC_FUNCTIONS -DLZ4_PUBLISH_STATIC_FUNCTIONS" $(MAKE) -C $(LZ4DIR) liblz4
+
lz4c: lz4
- ln -s lz4$(EXT) lz4c$(EXT)
+ $(LN_SF) lz4$(EXT) lz4c$(EXT)
lz4c32: CFLAGS += -m32
lz4c32 : $(SRCFILES)
@@ -94,28 +115,32 @@ lz4.1: lz4.1.md $(LIBVER_SRC)
man: lz4.1
clean-man:
- rm lz4.1
+ $(RM) lz4.1
preview-man: clean-man man
man ./lz4.1
clean:
+ifeq ($(WINBASED),yes)
+ $(Q)$(RM) *.rc
+endif
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
- lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4$(EXT) lz4cat$(EXT)
+ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) lz4-wlib$(EXT) \
+ unlz4$(EXT) lz4cat$(EXT)
@echo Cleaning completed
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD))
+ifeq ($(POSIX_ENV),Yes)
unlz4: lz4
- ln -s lz4$(EXT) unlz4$(EXT)
+ $(LN_SF) lz4$(EXT) unlz4$(EXT)
lz4cat: lz4
- ln -s lz4$(EXT) lz4cat$(EXT)
+ $(LN_SF) lz4$(EXT) lz4cat$(EXT)
DESTDIR ?=
# directory variables : GNU conventions prefer lowercase
@@ -134,28 +159,18 @@ mandir ?= $(MANDIR)
MAN1DIR ?= $(mandir)/man1
man1dir ?= $(MAN1DIR)
-ifneq (,$(filter $(shell uname),SunOS))
-INSTALL ?= ginstall
-else
-INSTALL ?= install
-endif
-
-INSTALL_PROGRAM ?= $(INSTALL) -m 755
-INSTALL_DATA ?= $(INSTALL) -m 644
-
-
install: lz4
@echo Installing binaries
- @$(INSTALL) -d -m 755 $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
+ @$(INSTALL_DIR) $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
@$(INSTALL_PROGRAM) lz4$(EXT) $(DESTDIR)$(bindir)/lz4$(EXT)
- @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT)
- @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT)
- @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT)
+ @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT)
+ @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT)
+ @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT)
@echo Installing man pages
@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1
- @ln -sf lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
- @ln -sf lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1
- @ln -sf lz4.1 $(DESTDIR)$(man1dir)/unlz4.1
+ @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
+ @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1
+ @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/unlz4.1
@echo lz4 installation completed
uninstall:
diff --git a/programs/README.md b/programs/README.md
index 2ad0449..c1995af 100644
--- a/programs/README.md
+++ b/programs/README.md
@@ -1,18 +1,26 @@
Command Line Interface for LZ4 library
============================================
-Command Line Interface (CLI) can be created using the `make` command without any additional parameters.
-There are also multiple targets that create different variations of CLI:
+### Build
+The Command Line Interface (CLI) can be generated
+using the `make` command without any additional parameters.
+
+The `Makefile` script supports all [standard conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html),
+including standard targets (`all`, `install`, `clean`, etc.)
+and standard variables (`CC`, `CFLAGS`, `CPPFLAGS`, etc.).
+
+For advanced use cases, there are targets to different variations of the CLI:
- `lz4` : default CLI, with a command line syntax close to gzip
- `lz4c` : Same as `lz4` with additional support legacy lz4 commands (incompatible with gzip)
- `lz4c32` : Same as `lz4c`, but forced to compile in 32-bits mode
+The CLI generates and decodes [LZ4-compressed frames](../doc/lz4_Frame_format.md).
+
#### Aggregation of parameters
CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`.
-
#### Benchmark in Command Line Interface
CLI includes in-memory compression benchmark module for lz4.
The benchmark is conducted using a given filename.
@@ -38,7 +46,9 @@ Arguments :
-9 : High compression
-d : decompression (default for .lz4 extension)
-z : force compression
+ -D FILE: use FILE as dictionary
-f : overwrite output without prompting
+ -k : preserve source files(s) (default)
--rm : remove source file(s) after successful de/compression
-h/-H : display help/long help and exit
@@ -51,17 +61,20 @@ Advanced arguments :
-m : multiple input files (implies automatic output filenames)
-r : operate recursively on directories (sets also -m)
-l : compress using Legacy format (Linux kernel compression)
- -B# : Block size [4-7] (default : 7)
+ -B# : cut file into blocks of size # bytes [32+]
+ or predefined block size [4-7] (default: 7)
-BD : Block dependency (improve compression ratio)
+ -BX : enable block checksum (default:disabled)
--no-frame-crc : disable stream checksum (default:enabled)
--content-size : compressed frame includes original size (default:not present)
--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)
+--favor-decSpeed: compressed files decompress faster, but are less compressed
+--fast[=#]: switch to ultra fast compression level (default: 1)
+
Benchmark arguments :
-b# : benchmark file(s), using # compression level (default : 1)
-e# : test all compression levels from -bX to # (default : 1)
- -i# : minimum evaluation time in seconds (default : 3s)
- -B# : cut file into independent blocks of size # bytes [32+]
- or predefined block size [4-7] (default: 7)
+ -i# : minimum evaluation time in seconds (default : 3s)```
```
#### License
diff --git a/programs/bench.c b/programs/bench.c
index 11bf044..3357d14 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -45,17 +45,176 @@
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"
+#include "bench.h"
-
+#define LZ4_STATIC_LINKING_ONLY
#include "lz4.h"
-#define COMPRESSOR0 LZ4_compress_local
-static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) {
- int const acceleration = (clevel < 0) ? -clevel + 1 : 1;
- return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration);
-}
+#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
-#define COMPRESSOR1 LZ4_compress_HC
-#define DEFAULTCOMPRESSOR COMPRESSOR0
+
+
+/* *************************************
+* Compression parameters and functions
+***************************************/
+
+struct compressionParameters
+{
+ int cLevel;
+ const char* dictBuf;
+ int dictSize;
+
+ LZ4_stream_t* LZ4_stream;
+ LZ4_stream_t* LZ4_dictStream;
+ LZ4_streamHC_t* LZ4_streamHC;
+ LZ4_streamHC_t* LZ4_dictStreamHC;
+
+ void (*initFunction)(
+ struct compressionParameters* pThis);
+ void (*resetFunction)(
+ const struct compressionParameters* pThis);
+ int (*blockFunction)(
+ const struct compressionParameters* pThis,
+ const char* src, char* dst, int srcSize, int dstSize);
+ void (*cleanupFunction)(
+ const struct compressionParameters* pThis);
+};
+
+static void LZ4_compressInitNoStream(
+ struct compressionParameters* pThis)
+{
+ pThis->LZ4_stream = NULL;
+ pThis->LZ4_dictStream = NULL;
+ pThis->LZ4_streamHC = NULL;
+ pThis->LZ4_dictStreamHC = NULL;
+}
+
+static void LZ4_compressInitStream(
+ struct compressionParameters* pThis)
+{
+ pThis->LZ4_stream = LZ4_createStream();
+ pThis->LZ4_dictStream = LZ4_createStream();
+ pThis->LZ4_streamHC = NULL;
+ pThis->LZ4_dictStreamHC = NULL;
+ LZ4_loadDict(pThis->LZ4_dictStream, pThis->dictBuf, pThis->dictSize);
+}
+
+static void LZ4_compressInitStreamHC(
+ struct compressionParameters* pThis)
+{
+ pThis->LZ4_stream = NULL;
+ pThis->LZ4_dictStream = NULL;
+ pThis->LZ4_streamHC = LZ4_createStreamHC();
+ pThis->LZ4_dictStreamHC = LZ4_createStreamHC();
+ LZ4_loadDictHC(pThis->LZ4_dictStreamHC, pThis->dictBuf, pThis->dictSize);
+}
+
+static void LZ4_compressResetNoStream(
+ const struct compressionParameters* pThis)
+{
+ (void)pThis;
+}
+
+static void LZ4_compressResetStream(
+ const struct compressionParameters* pThis)
+{
+ LZ4_resetStream_fast(pThis->LZ4_stream);
+ LZ4_attach_dictionary(pThis->LZ4_stream, pThis->LZ4_dictStream);
+}
+
+static void LZ4_compressResetStreamHC(
+ const struct compressionParameters* pThis)
+{
+ LZ4_resetStreamHC_fast(pThis->LZ4_streamHC, pThis->cLevel);
+ LZ4_attach_HC_dictionary(pThis->LZ4_streamHC, pThis->LZ4_dictStreamHC);
+}
+
+static int LZ4_compressBlockNoStream(
+ const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
+{
+ int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
+ return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration);
+}
+
+static int LZ4_compressBlockNoStreamHC(
+ const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
+{
+ return LZ4_compress_HC(src, dst, srcSize, dstSize, pThis->cLevel);
+}
+
+static int LZ4_compressBlockStream(
+ const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
+{
+ int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
+ return LZ4_compress_fast_continue(pThis->LZ4_stream, src, dst, srcSize, dstSize, acceleration);
+}
+
+static int LZ4_compressBlockStreamHC(
+ const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
+{
+ return LZ4_compress_HC_continue(pThis->LZ4_streamHC, src, dst, srcSize, dstSize);
+}
+
+static void LZ4_compressCleanupNoStream(
+ const struct compressionParameters* pThis)
+{
+ (void)pThis;
+}
+
+static void LZ4_compressCleanupStream(
+ const struct compressionParameters* pThis)
+{
+ LZ4_freeStream(pThis->LZ4_stream);
+ LZ4_freeStream(pThis->LZ4_dictStream);
+}
+
+static void LZ4_compressCleanupStreamHC(
+ const struct compressionParameters* pThis)
+{
+ LZ4_freeStreamHC(pThis->LZ4_streamHC);
+ LZ4_freeStreamHC(pThis->LZ4_dictStreamHC);
+}
+
+static void LZ4_buildCompressionParameters(
+ struct compressionParameters* pParams,
+ int cLevel, const char* dictBuf, int dictSize)
+{
+ pParams->cLevel = cLevel;
+ pParams->dictBuf = dictBuf;
+ pParams->dictSize = dictSize;
+
+ if (dictSize) {
+ if (cLevel < LZ4HC_CLEVEL_MIN) {
+ pParams->initFunction = LZ4_compressInitStream;
+ pParams->resetFunction = LZ4_compressResetStream;
+ pParams->blockFunction = LZ4_compressBlockStream;
+ pParams->cleanupFunction = LZ4_compressCleanupStream;
+ } else {
+ pParams->initFunction = LZ4_compressInitStreamHC;
+ pParams->resetFunction = LZ4_compressResetStreamHC;
+ pParams->blockFunction = LZ4_compressBlockStreamHC;
+ pParams->cleanupFunction = LZ4_compressCleanupStreamHC;
+ }
+ } else {
+ pParams->initFunction = LZ4_compressInitNoStream;
+ pParams->resetFunction = LZ4_compressResetNoStream;
+ pParams->cleanupFunction = LZ4_compressCleanupNoStream;
+
+ if (cLevel < LZ4HC_CLEVEL_MIN) {
+ pParams->blockFunction = LZ4_compressBlockNoStream;
+ } else {
+ pParams->blockFunction = LZ4_compressBlockNoStreamHC;
+ }
+ }
+}
+
#define LZ4_isError(errcode) (errcode==0)
@@ -79,6 +238,8 @@ static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSi
#define MB *(1 <<20)
#define GB *(1U<<30)
+#define LZ4_MAX_DICT_SIZE (64 KB)
+
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
static U32 g_compressibilityDefault = 50;
@@ -152,17 +313,13 @@ typedef struct {
size_t resSize;
} blockParam_t;
-struct compressionParameters
-{
- int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
-};
-
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b))
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
- const size_t* fileSizes, U32 nbFiles)
+ const size_t* fileSizes, U32 nbFiles,
+ const char* dictBuf, int dictSize)
{
size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
@@ -172,27 +329,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
void* const resultBuffer = malloc(srcSize);
U32 nbBlocks;
struct compressionParameters compP;
- int cfunctionId;
/* checks */
if (!compressedBuffer || !resultBuffer || !blockTable)
EXM_THROW(31, "allocation error : not enough memory");
- /* init */
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
- /* Init */
- if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1;
- switch (cfunctionId)
- {
-#ifdef COMPRESSOR0
- case 0 : compP.compressionFunction = COMPRESSOR0; break;
-#endif
-#ifdef COMPRESSOR1
- case 1 : compP.compressionFunction = COMPRESSOR1; break;
-#endif
- default : compP.compressionFunction = DEFAULTCOMPRESSOR;
- }
+ /* init */
+ LZ4_buildCompressionParameters(&compP, cLevel, dictBuf, dictSize);
+ compP.initFunction(&compP);
/* Init blockTable data */
{ const char* srcPtr = (const char*)srcBuffer;
@@ -209,7 +355,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
blockTable[nbBlocks].cPtr = cPtr;
blockTable[nbBlocks].resPtr = resPtr;
blockTable[nbBlocks].srcSize = thisBlockSize;
- blockTable[nbBlocks].cRoom = LZ4_compressBound((int)thisBlockSize);
+ blockTable[nbBlocks].cRoom = (size_t)LZ4_compressBound((int)thisBlockSize);
srcPtr += thisBlockSize;
cPtr += blockTable[nbBlocks].cRoom;
resPtr += thisBlockSize;
@@ -256,9 +402,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
U32 nbLoops;
for (nbLoops=0; nbLoops < nbCompressionLoops; nbLoops++) {
U32 blockNb;
+ compP.resetFunction(&compP);
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
- size_t const rSize = compP.compressionFunction(blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr, (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom, cLevel);
- if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4_compress() failed");
+ size_t const rSize = (size_t)compP.blockFunction(
+ &compP,
+ blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr,
+ (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom);
+ if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4 compression failed");
blockTable[blockNb].cSize = rSize;
} }
{ U64 const clockSpan = UTIL_clockSpanNano(clockStart);
@@ -298,12 +448,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
- size_t const regenSize = LZ4_decompress_safe(blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize);
- if (LZ4_isError(regenSize)) {
- DISPLAY("LZ4_decompress_safe() failed on block %u \n", blockNb);
+ int const regenSize = LZ4_decompress_safe_usingDict(
+ blockTable[blockNb].cPtr, blockTable[blockNb].resPtr,
+ (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize,
+ dictBuf, dictSize);
+ if (regenSize < 0) {
+ DISPLAY("LZ4_decompress_safe_usingDict() failed on block %u \n", blockNb);
break;
}
- blockTable[blockNb].resSize = regenSize;
+ blockTable[blockNb].resSize = (size_t)regenSize;
} }
{ U64 const clockSpan = UTIL_clockSpanNano(clockStart);
if (clockSpan > 0) {
@@ -364,6 +517,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
} /* Bench */
/* clean up */
+ compP.cleanupFunction(&compP);
free(blockTable);
free(compressedBuffer);
free(resultBuffer);
@@ -397,7 +551,8 @@ static size_t BMK_findMaxMem(U64 requiredMem)
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast,
- const size_t* fileSizes, unsigned nbFiles)
+ const size_t* fileSizes, unsigned nbFiles,
+ const char* dictBuf, int dictSize)
{
int l;
@@ -415,7 +570,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
- fileSizes, nbFiles);
+ fileSizes, nbFiles,
+ dictBuf, dictSize);
}
}
@@ -456,7 +612,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
- int cLevel, int cLevelLast)
+ int cLevel, int cLevelLast,
+ const char* dictBuf, int dictSize)
{
void* srcBuffer;
size_t benchedSize;
@@ -488,7 +645,8 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
{ const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast,
- fileSizes, nbFiles);
+ fileSizes, nbFiles,
+ dictBuf, dictSize);
}
/* clean up */
@@ -497,7 +655,8 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
}
-static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
+static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility,
+ const char* dictBuf, int dictSize)
{
char name[20] = {0};
size_t benchedSize = 10000000;
@@ -511,7 +670,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
/* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
- BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1);
+ BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, dictBuf, dictSize);
/* clean up */
free(srcBuffer);
@@ -519,7 +678,8 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
- int cLevel, int cLevelLast)
+ int cLevel, int cLevelLast,
+ const char* dictBuf, int dictSize)
{
unsigned fileNb;
if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
@@ -528,29 +688,59 @@ int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
for (fileNb=0; fileNb<nbFiles; fileNb++)
- BMK_benchFileTable(fileNamesTable+fileNb, 1, cLevel, cLevelLast);
+ BMK_benchFileTable(fileNamesTable+fileNb, 1, cLevel, cLevelLast, dictBuf, dictSize);
return 0;
}
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
- int cLevel, int cLevelLast)
+ int cLevel, int cLevelLast,
+ const char* dictFileName)
{
double const compressibility = (double)g_compressibilityDefault / 100;
+ char* dictBuf = NULL;
+ int dictSize = 0;
if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+ if (dictFileName) {
+ FILE* dictFile = NULL;
+ U64 dictFileSize = UTIL_getFileSize(dictFileName);
+ if (!dictFileSize) EXM_THROW(25, "Dictionary error : could not stat dictionary file");
+
+ dictFile = fopen(dictFileName, "rb");
+ if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
+
+ if (dictFileSize > LZ4_MAX_DICT_SIZE) {
+ dictSize = LZ4_MAX_DICT_SIZE;
+ if (UTIL_fseek(dictFile, dictFileSize - dictSize, SEEK_SET))
+ EXM_THROW(25, "Dictionary error : could not seek dictionary file");
+ } else {
+ dictSize = (int)dictFileSize;
+ }
+
+ dictBuf = (char *)malloc(dictSize);
+ if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
+
+ if (fread(dictBuf, 1, dictSize, dictFile) != (size_t)dictSize)
+ EXM_THROW(25, "Dictionary error : could not read dictionary file");
+
+ fclose(dictFile);
+ }
+
if (nbFiles == 0)
- BMK_syntheticTest(cLevel, cLevelLast, compressibility);
+ BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, dictSize);
else {
if (g_benchSeparately)
- BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast);
+ BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize);
else
- BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast);
+ BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize);
}
+
+ free(dictBuf);
return 0;
}
diff --git a/programs/bench.h b/programs/bench.h
index bb67bee..22ebf60 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -26,7 +26,8 @@
#include <stddef.h>
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
- int cLevel, int cLevelLast);
+ int cLevel, int cLevelLast,
+ const char* dictFileName);
/* Set Parameters */
void BMK_setNbSeconds(unsigned nbLoops);
diff --git a/programs/lz4-exe.rc.in b/programs/lz4-exe.rc.in
new file mode 100644
index 0000000..7b81030
--- /dev/null
+++ b/programs/lz4-exe.rc.in
@@ -0,0 +1,27 @@
+1 VERSIONINFO
+FILEVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0
+PRODUCTVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0
+FILEFLAGSMASK 0
+FILEOS 0x40000
+FILETYPE 1
+{
+ BLOCK "StringFileInfo"
+ {
+ BLOCK "040904B0"
+ {
+ VALUE "CompanyName", "Yann Collet"
+ VALUE "FileDescription", "Extremely fast compression"
+ VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
+ VALUE "InternalName", "@PROGNAME@"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "OriginalFilename", "@PROGNAME@.@EXT@"
+ VALUE "ProductName", "LZ4"
+ VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
+ }
+ }
+ BLOCK "VarFileInfo"
+ {
+ VALUE "Translation", 0x0409, 1200
+ }
+}
+
diff --git a/programs/lz4.1 b/programs/lz4.1
index f35e29d..d758ed5 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -1,5 +1,5 @@
.
-.TH "LZ4" "1" "September 2018" "lz4 1.8.3" "User Commands"
+.TH "LZ4" "1" "July 2019" "lz4 1.9.2" "User Commands"
.
.SH "NAME"
\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files
@@ -23,9 +23,6 @@ When writing scripts that need to decompress files, it is recommended to always
\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are :
.
.IP "\(bu" 4
-\fBlz4\fR preserves original files
-.
-.IP "\(bu" 4
\fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files)
.
.IP "\(bu" 4
@@ -35,19 +32,16 @@ When writing scripts that need to decompress files, it is recommended to always
\fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression)
.
.IP "\(bu" 4
-\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them)
-.
-.IP "\(bu" 4
-If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\.
+\fBlz4\fR preserves original files
.
.IP "\(bu" 4
-If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\.
+\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them)
.
.IP "\(bu" 4
-As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create \fBfile\.lz4\fR\.
+When no destination is specified, result is sent on implicit output, which depends on \fBstdout\fR status\. When \fBstdout\fR \fIis Not the console\fR, it becomes the implicit output\. Otherwise, if \fBstdout\fR is the console, the implicit output is \fBfilename\.lz4\fR\.
.
.IP "\(bu" 4
-Another consequence of those rules is that to run \fBlz4\fR under \fBnohup\fR, you should provide a destination file: \fBnohup lz4 file file\.lz4\fR, because \fBnohup\fR writes the specified command\'s output to a file\.
+It is considered bad practice to rely on implicit output in scripts\. because the script\'s environment may change\. Always use explicit output in scripts\. \fB\-c\fR ensures that output will be \fBstdout\fR\. Conversely, providing a destination name, or using \fB\-m\fR ensures that the output will be either the specified name, or \fBfilename\.lz4\fR respectively\.
.
.IP "" 0
.
@@ -55,7 +49,7 @@ Another consequence of those rules is that to run \fBlz4\fR under \fBnohup\fR, y
Default behaviors can be modified by opt\-in commands, detailed below\.
.
.IP "\(bu" 4
-\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\.
+\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications become disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\.
.
.IP "\(bu" 4
Similarly, \fBlz4 \-m \-d\fR can decompress multiple \fB*\.lz4\fR files\.
@@ -111,6 +105,10 @@ Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is di
\fB\-b#\fR
Benchmark mode, using \fB#\fR compression level\.
.
+.TP
+\fB\-\-list\fR
+List information about \.lz4 files\. note : current implementation is limited to single\-frame \.lz4 files\.
+.
.SS "Operation modifiers"
.
.TP
@@ -118,6 +116,18 @@ Benchmark mode, using \fB#\fR compression level\.
Compression level, with # being any value from 1 to 12\. Higher values trade compression speed for compression ratio\. Values above 12 are considered the same as 12\. Recommended values are 1 for fast compression (default), and 9 for high compression\. Speed/compression trade\-off will vary depending on data to compress\. Decompression speed remains fast at all settings\.
.
.TP
+\fB\-\-fast[=#]\fR
+Switch to ultra\-fast compression levels\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. If \fB=#\fR is not present, it defaults to \fB1\fR\. This setting overrides compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
+.
+.TP
+\fB\-\-best\fR
+Set highest compression level\. Same as -12\.
+.
+.TP
+\fB\-\-favor\-decSpeed\fR
+Generate compressed data optimized for decompression speed\. Compressed data will be larger as a consequence (typically by ~0\.5%), while decompression speed will be improved by 5\-20%, depending on use cases\. This option only works in combination with very high compression levels (>=10)\.
+.
+.TP
\fB\-D dictionaryName\fR
Compress, decompress or benchmark using dictionary \fIdictionaryName\fR\. Compression and decompression must use the same dictionary to be compatible\. Using a different dictionary during decompression will either abort due to decompression error, or generate a checksum error\.
.
@@ -137,7 +147,7 @@ Force write to standard output, even if it is the console\.
.
.TP
\fB\-m\fR \fB\-\-multiple\fR
-Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\.
+Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. Can also be used to list multiple files\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\.
.
.TP
\fB\-r\fR
@@ -151,12 +161,12 @@ Block size [4\-7](default : 7)
\fB\-B4\fR= 64KB ; \fB\-B5\fR= 256KB ; \fB\-B6\fR= 1MB ; \fB\-B7\fR= 4MB
.
.TP
-\fB\-BD\fR
-Block Dependency (improves compression ratio on small blocks)
+\fB\-BI\fR
+Produce independent blocks (default)
.
.TP
-\fB\-\-fast[=#]\fR
-switch to ultra\-fast compression levels\. If \fB=#\fR is not present, it defaults to \fB1\fR\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. This setting overwrites compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
+\fB\-BD\fR
+Blocks depend on predecessors (improves compression ratio, more noticeable on small blocks)
.
.TP
\fB\-\-[no\-]frame\-crc\fR
diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index 12b8e29..56c0053 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -31,29 +31,29 @@ The native file format is the `.lz4` format.
`lz4` supports a command line syntax similar _but not identical_ to `gzip(1)`.
Differences are :
- * `lz4` preserves original files
* `lz4` compresses a single file by default (see `-m` for multiple files)
* `lz4 file1 file2` means : compress file1 _into_ file2
* `lz4 file.lz4` will default to decompression (use `-z` to force compression)
+ * `lz4` preserves original files
* `lz4` shows real-time notification statistics
during compression or decompression of a single file
(use `-q` to silence them)
- * If no destination name is provided, result is sent to `stdout`
- _except if stdout is the console_.
- * If no destination name is provided, __and__ if `stdout` is the console,
- `file` is compressed into `file.lz4`.
- * As a consequence of previous rules, note the following example :
- `lz4 file | consumer` sends compressed data to `consumer` through `stdout`,
- hence it does _not_ create `file.lz4`.
- * Another consequence of those rules is that to run `lz4` under `nohup`,
- you should provide a destination file: `nohup lz4 file file.lz4`,
- because `nohup` writes the specified command's output to a file.
+ * When no destination is specified, result is sent on implicit output,
+ which depends on `stdout` status.
+ When `stdout` _is Not the console_, it becomes the implicit output.
+ Otherwise, if `stdout` is the console, the implicit output is `filename.lz4`.
+ * It is considered bad practice to rely on implicit output in scripts.
+ because the script's environment may change.
+ Always use explicit output in scripts.
+ `-c` ensures that output will be `stdout`.
+ Conversely, providing a destination name, or using `-m`
+ ensures that the output will be either the specified name, or `filename.lz4` respectively.
Default behaviors can be modified by opt-in commands, detailed below.
* `lz4 -m` makes it possible to provide multiple input filenames,
which will be compressed into files using suffix `.lz4`.
- Progress notifications are also disabled by default (use `-v` to enable them).
+ Progress notifications become disabled by default (use `-v` to enable them).
This mode has a behavior which more closely mimics `gzip` command line,
with the main remaining difference being that source files are preserved by default.
* Similarly, `lz4 -m -d` can decompress multiple `*.lz4` files.
@@ -81,8 +81,7 @@ In some cases, some options can be expressed using short command `-x`
or long command `--long-word`.
Short commands can be concatenated together.
For example, `-d -c` is equivalent to `-dc`.
-Long commands cannot be concatenated.
-They must be clearly separated by a space.
+Long commands cannot be concatenated. They must be clearly separated by a space.
### Multiple commands
@@ -114,6 +113,10 @@ only the latest one will be applied.
* `-b#`:
Benchmark mode, using `#` compression level.
+* `--list`:
+ List information about .lz4 files.
+ note : current implementation is limited to single-frame .lz4 files.
+
### Operation modifiers
* `-#`:
@@ -126,12 +129,21 @@ only the latest one will be applied.
Decompression speed remains fast at all settings.
* `--fast[=#]`:
- switch to ultra-fast compression levels.
+ Switch to ultra-fast compression levels.
The higher the value, the faster the compression speed, at the cost of some compression ratio.
If `=#` is not present, it defaults to `1`.
This setting overrides compression level if one was set previously.
Similarly, if a compression level is set after `--fast`, it overrides it.
+* `--best`:
+ Set highest compression level. Same as -12.
+
+* `--favor-decSpeed`:
+ Generate compressed data optimized for decompression speed.
+ Compressed data will be larger as a consequence (typically by ~0.5%),
+ while decompression speed will be improved by 5-20%, depending on use cases.
+ This option only works in combination with very high compression levels (>=10).
+
* `-D dictionaryName`:
Compress, decompress or benchmark using dictionary _dictionaryName_.
Compression and decompression must use the same dictionary to be compatible.
@@ -155,6 +167,7 @@ only the latest one will be applied.
Multiple input files.
Compressed file names will be appended a `.lz4` suffix.
This mode also reduces notification level.
+ Can also be used to list multiple files.
`lz4 -m` has a behavior equivalent to `gzip -k`
(it preserves source files by default).
@@ -166,8 +179,11 @@ only the latest one will be applied.
Block size \[4-7\](default : 7)<br/>
`-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB
+* `-BI`:
+ Produce independent blocks (default)
+
* `-BD`:
- Block Dependency (improves compression ratio on small blocks)
+ Blocks depend on predecessors (improves compression ratio, more noticeable on small blocks)
* `--[no-]frame-crc`:
Select frame checksum (default:enabled)
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 26a8089..523b8a8 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -67,6 +67,7 @@ static int g_lz4c_legacy_commands = 0;
/*-************************************
* Macros
***************************************/
+#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */
@@ -92,8 +93,11 @@ static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : dow
***************************************/
#define DEFAULT_COMPRESSOR LZ4IO_compressFilename
#define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename
-int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */
-
+int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel, const LZ4IO_prefs_t* prefs); /* hidden function */
+int LZ4IO_compressMultipleFilenames_Legacy(
+ const char** inFileNamesTable, int ifntSize,
+ const char* suffix,
+ int compressionLevel, const LZ4IO_prefs_t* prefs);
/*-***************************
* Functions
@@ -134,20 +138,22 @@ static int usage_advanced(const char* exeName)
DISPLAY( " -r : operate recursively on directories (sets also -m) \n");
#endif
DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n");
- DISPLAY( " -B# : Block size [4-7] (default : 7) \n");
- DISPLAY( " -BD : Block dependency (improve compression ratio) \n");
+ DISPLAY( " -B# : cut file into blocks of size # bytes [32+] \n");
+ DISPLAY( " or predefined block size [4-7] (default: 7) \n");
+ DISPLAY( " -BI : Block Independence (default) \n");
+ DISPLAY( " -BD : Block dependency (improves compression ratio) \n");
DISPLAY( " -BX : enable block checksum (default:disabled) \n");
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n");
DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
+ DISPLAY( "--list FILE : lists information about .lz4 files (useful for files compressed with --content-size flag)\n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n");
- DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
+ DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %i)\n", 1);
+ DISPLAY( "--best : same as -%d\n", LZ4HC_CLEVEL_MAX);
DISPLAY( "Benchmark arguments : \n");
DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n");
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n");
- DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+] \n");
- DISPLAY( " or predefined block size [4-7] (default: 7) \n");
if (g_lz4c_legacy_commands) {
DISPLAY( "Legacy arguments : \n");
DISPLAY( " -c0 : fast compression \n");
@@ -251,16 +257,16 @@ static int exeNameMatch(const char* exeName, const char* test)
}
/*! readU32FromChar() :
- @return : unsigned integer value read from input in `char` format
- allows and interprets K, KB, KiB, M, MB and MiB suffix.
- Will also modify `*stringPtr`, advancing it to position where it stopped reading.
- Note : function result can overflow if digit string > MAX_UINT */
+ * @return : unsigned integer value read from input in `char` format
+ * allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ * Will also modify `*stringPtr`, advancing it to position where it stopped reading.
+ * Note : function result can overflow if digit string > MAX_UINT */
static unsigned readU32FromChar(const char** stringPtr)
{
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
result *= 10;
- result += **stringPtr - '0';
+ result += (unsigned)(**stringPtr - '0');
(*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
@@ -278,7 +284,7 @@ static unsigned readU32FromChar(const char** stringPtr)
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
* @return 0 and doesn't modify *stringPtr otherwise.
*/
-static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+static int longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
@@ -286,7 +292,20 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
return result;
}
-typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e;
+typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench, om_list } operationMode_e;
+
+/** determineOpMode() :
+ * auto-determine operation mode, based on input filename extension
+ * @return `om_decompress` if input filename has .lz4 extension and `om_compress` otherwise.
+ */
+static operationMode_e determineOpMode(const char* inputFilename)
+{
+ size_t const inSize = strlen(inputFilename);
+ size_t const extSize = strlen(LZ4_EXTENSION);
+ size_t const extStart= (inSize > extSize) ? inSize-extSize : 0;
+ if (!strcmp(inputFilename+extStart, LZ4_EXTENSION)) return om_decompress;
+ else return om_compress;
+}
int main(int argc, const char** argv)
{
@@ -304,11 +323,12 @@ int main(int argc, const char** argv)
const char* output_filename= NULL;
const char* dictionary_filename = NULL;
char* dynNameSpace = NULL;
- const char** inFileNames = (const char**) calloc(argc, sizeof(char*));
+ const char** inFileNames = (const char**)calloc((size_t)argc, sizeof(char*));
unsigned ifnIdx=0;
+ LZ4IO_prefs_t* const prefs = LZ4IO_defaultPreferences();
const char nullOutput[] = NULL_OUTPUT;
const char extension[] = LZ4_EXTENSION;
- size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
+ size_t blockSize = LZ4IO_setBlockSizeID(prefs, LZ4_BLOCKSIZEID_DEFAULT);
const char* const exeName = lastNameFromPath(argv[0]);
#ifdef UTIL_HAS_CREATEFILELIST
const char** extendedFileList = NULL;
@@ -322,13 +342,14 @@ int main(int argc, const char** argv)
return 1;
}
inFileNames[0] = stdinmark;
- LZ4IO_setOverwrite(0);
+ LZ4IO_setOverwrite(prefs, 0);
/* predefined behaviors, based on binary/link name */
if (exeNameMatch(exeName, LZ4CAT)) {
mode = om_decompress;
- LZ4IO_setOverwrite(1);
- LZ4IO_setRemoveSrcFile(0);
+ LZ4IO_setOverwrite(prefs, 1);
+ LZ4IO_setPassThrough(prefs, 1);
+ LZ4IO_setRemoveSrcFile(prefs, 0);
forceStdout=1;
output_filename=stdoutmark;
displayLevel=1;
@@ -360,23 +381,24 @@ int main(int argc, const char** argv)
|| (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; continue; }
if (!strcmp(argument, "--test")) { mode = om_test; continue; }
- if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; }
- if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; }
+ if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(prefs, 1); continue; }
+ if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(prefs, 0); continue; }
if ((!strcmp(argument, "--stdout"))
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; continue; }
- if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; }
- if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
- if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; }
- if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
- if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; }
- if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
- if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(1); continue; }
+ if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); continue; }
+ if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); continue; }
+ if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(prefs, 1); continue; }
+ if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(prefs, 0); continue; }
+ if (!strcmp(argument, "--list")) { mode = om_list; continue; }
+ if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(prefs, 2); continue; }
+ if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(prefs, 0); continue; }
+ if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(prefs, 1); continue; }
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; }
- if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
+ if (!strcmp(argument, "--version")) { DISPLAYOUT(WELCOME_MESSAGE); goto _cleanup; }
if (!strcmp(argument, "--help")) { usage_advanced(exeName); goto _cleanup; }
- if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(0); continue; } /* keep source file (default) */
- if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(1); continue; }
+ if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(prefs, 0); continue; } /* keep source file (default) */
+ if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(prefs, 1); continue; }
if (longCommandWArg(&argument, "--fast")) {
/* Parse optional acceleration factor */
if (*argument == '=') {
@@ -396,6 +418,9 @@ int main(int argc, const char** argv)
}
continue;
}
+
+ /* For gzip(1) compatibility */
+ if (!strcmp(argument, "--best")) { cLevel=LZ4HC_CLEVEL_MAX; continue; }
}
while (argument[1]!=0) {
@@ -407,11 +432,11 @@ int main(int argc, const char** argv)
if (!strcmp(argument, "c1")) { cLevel=9; argument++; continue; } /* -c1 (high compression) */
if (!strcmp(argument, "c2")) { cLevel=12; argument++; continue; } /* -c2 (very high compression) */
if (!strcmp(argument, "hc")) { cLevel=12; argument++; continue; } /* -hc (very high compression) */
- if (!strcmp(argument, "y")) { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */
+ if (!strcmp(argument, "y")) { LZ4IO_setOverwrite(prefs, 1); continue; } /* -y (answer 'yes' to overwrite permission) */
}
if ((*argument>='0') && (*argument<='9')) {
- cLevel = readU32FromChar(&argument);
+ cLevel = (int)readU32FromChar(&argument);
argument--;
continue;
}
@@ -420,13 +445,13 @@ int main(int argc, const char** argv)
switch(argument[0])
{
/* Display help */
- case 'V': DISPLAY(WELCOME_MESSAGE); goto _cleanup; /* Version */
+ case 'V': DISPLAYOUT(WELCOME_MESSAGE); goto _cleanup; /* Version */
case 'h': usage_advanced(exeName); goto _cleanup;
case 'H': usage_longhelp(exeName); goto _cleanup;
case 'e':
argument++;
- cLevelLast = readU32FromChar(&argument);
+ cLevelLast = (int)readU32FromChar(&argument);
argument--;
break;
@@ -456,13 +481,17 @@ int main(int argc, const char** argv)
case 'd': mode = om_decompress; break;
/* Force stdout, even if stdout==console */
- case 'c': forceStdout=1; output_filename=stdoutmark; break;
+ case 'c':
+ forceStdout=1;
+ output_filename=stdoutmark;
+ LZ4IO_setPassThrough(prefs, 1);
+ break;
/* Test integrity */
case 't': mode = om_test; break;
/* Overwrite */
- case 'f': LZ4IO_setOverwrite(1); break;
+ case 'f': LZ4IO_setOverwrite(prefs, 1); break;
/* Verbose mode */
case 'v': displayLevel++; break;
@@ -471,7 +500,7 @@ int main(int argc, const char** argv)
case 'q': if (displayLevel) displayLevel--; break;
/* keep source file (default anyway, so useless) (for xz/lzma compatibility) */
- case 'k': LZ4IO_setRemoveSrcFile(0); break;
+ case 'k': LZ4IO_setRemoveSrcFile(prefs, 0); break;
/* Modify Block Properties */
case 'B':
@@ -479,8 +508,9 @@ int main(int argc, const char** argv)
int exitBlockProperties=0;
switch(argument[1])
{
- case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break;
- case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* disabled by default */
+ case 'D': LZ4IO_setBlockMode(prefs, LZ4IO_blockLinked); argument++; break;
+ case 'I': LZ4IO_setBlockMode(prefs, LZ4IO_blockIndependent); argument++; break;
+ case 'X': LZ4IO_setBlockChecksumMode(prefs, 1); argument ++; break; /* disabled by default */
default :
if (argument[1] < '0' || argument[1] > '9') {
exitBlockProperties=1;
@@ -492,16 +522,17 @@ int main(int argc, const char** argv)
argument--;
if (B < 4) badusage(exeName);
if (B <= 7) {
- blockSize = LZ4IO_setBlockSizeID(B);
+ blockSize = LZ4IO_setBlockSizeID(prefs, B);
BMK_setBlockSize(blockSize);
DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
} else {
if (B < 32) badusage(exeName);
- BMK_setBlockSize(B);
- if (B >= 1024) {
- DISPLAYLEVEL(2, "bench: using blocks of size %u KB \n", (U32)(B>>10));
+ blockSize = LZ4IO_setBlockSize(prefs, B);
+ BMK_setBlockSize(blockSize);
+ if (blockSize >= 1024) {
+ DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
} else {
- DISPLAYLEVEL(2, "bench: using blocks of size %u bytes \n", (U32)(B));
+ DISPLAYLEVEL(2, "using blocks of size %u bytes \n", (U32)(blockSize));
}
}
break;
@@ -597,27 +628,27 @@ int main(int argc, const char** argv)
#endif
}
+ if (dictionary_filename) {
+ if (!strcmp(dictionary_filename, stdinmark) && IS_CONSOLE(stdin)) {
+ DISPLAYLEVEL(1, "refusing to read from a console\n");
+ exit(1);
+ }
+ LZ4IO_setDictionaryFilename(prefs, dictionary_filename);
+ }
+
/* benchmark and test modes */
if (mode == om_bench) {
BMK_setNotificationLevel(displayLevel);
- operationResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel, cLevelLast);
+ operationResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel, cLevelLast, dictionary_filename);
goto _cleanup;
}
if (mode == om_test) {
- LZ4IO_setTestMode(1);
+ LZ4IO_setTestMode(prefs, 1);
output_filename = nulmark;
mode = om_decompress; /* defer to decompress */
}
- if (dictionary_filename) {
- if (!strcmp(dictionary_filename, stdinmark) && IS_CONSOLE(stdin)) {
- DISPLAYLEVEL(1, "refusing to read from a console\n");
- exit(1);
- }
- LZ4IO_setDictionaryFilename(dictionary_filename);
- }
-
/* compress or decompress */
if (!input_filename) input_filename = stdinmark;
/* Check if input is defined as console; trigger an error in this case */
@@ -625,19 +656,34 @@ int main(int argc, const char** argv)
DISPLAYLEVEL(1, "refusing to read from a console\n");
exit(1);
}
- /* if input==stdin and no output defined, stdout becomes default output */
- if (!strcmp(input_filename, stdinmark) && !output_filename)
- output_filename = stdoutmark;
+ if (!strcmp(input_filename, stdinmark)) {
+ /* if input==stdin and no output defined, stdout becomes default output */
+ if (!output_filename) output_filename = stdoutmark;
+ }
+ else{
+#ifdef UTIL_HAS_CREATEFILELIST
+ if (!recursive && !UTIL_isRegFile(input_filename)) {
+#else
+ if (!UTIL_isRegFile(input_filename)) {
+#endif
+ DISPLAYLEVEL(1, "%s: is not a regular file \n", input_filename);
+ exit(1);
+ }
+ }
/* No output filename ==> try to select one automatically (when possible) */
while ((!output_filename) && (multiple_inputs==0)) {
- if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */
+ if (!IS_CONSOLE(stdout) && mode != om_list) {
+ /* Default to stdout whenever stdout is not the console.
+ * Note : this policy may change in the future, therefore don't rely on it !
+ * To ensure `stdout` is explicitly selected, use `-c` command flag.
+ * Conversely, to ensure output will not become `stdout`, use `-m` command flag */
+ DISPLAYLEVEL(1, "Warning : using stdout as default output. Do not rely on this behavior: use explicit `-c` instead ! \n");
+ output_filename=stdoutmark;
+ break;
+ }
if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */
- size_t const inSize = strlen(input_filename);
- size_t const extSize = strlen(LZ4_EXTENSION);
- size_t const extStart= (inSize > extSize) ? inSize-extSize : 0;
- if (!strcmp(input_filename+extStart, LZ4_EXTENSION)) mode = om_decompress;
- else mode = om_compress;
+ mode = determineOpMode(input_filename);
}
if (mode == om_compress) { /* compression to file */
size_t const l = strlen(input_filename);
@@ -665,35 +711,67 @@ int main(int argc, const char** argv)
break;
}
- /* Check if output is defined as console; trigger an error in this case */
+ if (mode == om_list){
+ /* Exit if trying to read from stdin as this isn't supported in this mode */
+ if(!strcmp(input_filename, stdinmark)){
+ DISPLAYLEVEL(1, "refusing to read from standard input in --list mode\n");
+ exit(1);
+ }
+ if(!multiple_inputs){
+ inFileNames[ifnIdx++] = input_filename;
+ }
+ }
+ else{
+ if (multiple_inputs==0) assert(output_filename);
+ }
+ /* when multiple_inputs==1, output_filename may simply be useless,
+ * however, output_filename must be !NULL for next strcmp() tests */
if (!output_filename) output_filename = "*\\dummy^!//";
+
+ /* Check if output is defined as console; trigger an error in this case */
if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) {
- DISPLAYLEVEL(1, "refusing to write to console without -c\n");
+ DISPLAYLEVEL(1, "refusing to write to console without -c \n");
exit(1);
}
/* Downgrade notification level in stdout and multiple file mode */
if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((multiple_inputs) && (displayLevel==2)) displayLevel=1;
+ /* Auto-determine compression or decompression, based on file extension */
+ if (mode == om_auto) {
+ mode = determineOpMode(input_filename);
+ }
+
/* IO Stream/File */
- LZ4IO_setNotificationLevel(displayLevel);
+ LZ4IO_setNotificationLevel((int)displayLevel);
if (ifnIdx == 0) multiple_inputs = 0;
if (mode == om_decompress) {
- if (multiple_inputs)
- operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION);
- else
- operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename);
+ if (multiple_inputs) {
+ const char* const dec_extension = !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION;
+ assert(ifnIdx <= INT_MAX);
+ operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, (int)ifnIdx, dec_extension, prefs);
+ } else {
+ operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename, prefs);
+ }
+ } else if (mode == om_list){
+ operationResult = LZ4IO_displayCompressedFilesInfo(inFileNames, ifnIdx);
} else { /* compression is default action */
if (legacy_format) {
DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n");
- LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
+ if(multiple_inputs){
+ const char* const leg_extension = !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION;
+ LZ4IO_compressMultipleFilenames_Legacy(inFileNames, (int)ifnIdx, leg_extension, cLevel, prefs);
+ } else {
+ LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel, prefs);
+ }
} else {
- if (multiple_inputs)
- operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel);
- else
- operationResult = DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
- }
- }
+ if (multiple_inputs) {
+ const char* const comp_extension = !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION;
+ assert(ifnIdx <= INT_MAX);
+ operationResult = LZ4IO_compressMultipleFilenames(inFileNames, (int)ifnIdx, comp_extension, cLevel, prefs);
+ } else {
+ operationResult = DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel, prefs);
+ } } }
_cleanup:
if (main_pause) waitEnter();
@@ -704,6 +782,7 @@ _cleanup:
inFileNames = NULL;
}
#endif
+ LZ4IO_freePreferences(prefs);
free((void*)inFileNames);
return operationResult;
}
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 28d6537..a274798 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -53,11 +53,11 @@
#include <time.h> /* clock */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
-#include "lz4io.h"
#include "lz4.h" /* still required for legacy format */
#include "lz4hc.h" /* still required for legacy format */
#define LZ4F_STATIC_LINKING_ONLY
#include "lz4frame.h"
+#include "lz4io.h"
/*****************************
@@ -90,6 +90,7 @@
* Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
@@ -102,23 +103,29 @@ static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result
} }
static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_time = 0;
+#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
/**************************************
* Local Parameters
**************************************/
-static int g_overwrite = 1;
-static int g_testMode = 0;
-static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT;
-static int g_blockChecksum = 0;
-static int g_streamChecksum = 1;
-static int g_blockIndependence = 1;
-static int g_sparseFileSupport = 1;
-static int g_contentSizeFlag = 0;
-static int g_useDictionary = 0;
-static unsigned g_favorDecSpeed = 0;
-static const char* g_dictionaryFilename = NULL;
+struct LZ4IO_prefs_s {
+ int passThrough;
+ int overwrite;
+ int testMode;
+ int blockSizeId;
+ size_t blockSize;
+ int blockChecksum;
+ int streamChecksum;
+ int blockIndependence;
+ int sparseFileSupport;
+ int contentSizeFlag;
+ int useDictionary;
+ unsigned favorDecSpeed;
+ const char* dictionaryFilename;
+ int removeSrcFile;
+};
/**************************************
* Exceptions
@@ -150,55 +157,110 @@ static const char* g_dictionaryFilename = NULL;
/* ****************** Parameters ******************** */
/* ************************************************** */
-int LZ4IO_setDictionaryFilename(const char* dictionaryFilename) {
- g_dictionaryFilename = dictionaryFilename;
- g_useDictionary = dictionaryFilename != NULL;
- return g_useDictionary;
+LZ4IO_prefs_t* LZ4IO_defaultPreferences(void)
+{
+ LZ4IO_prefs_t* const ret = (LZ4IO_prefs_t*)malloc(sizeof(*ret));
+ if (!ret) EXM_THROW(21, "Allocation error : not enough memory");
+ ret->passThrough = 0;
+ ret->overwrite = 1;
+ ret->testMode = 0;
+ ret->blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT;
+ ret->blockSize = 0;
+ ret->blockChecksum = 0;
+ ret->streamChecksum = 1;
+ ret->blockIndependence = 1;
+ ret->sparseFileSupport = 1;
+ ret->contentSizeFlag = 0;
+ ret->useDictionary = 0;
+ ret->favorDecSpeed = 0;
+ ret->dictionaryFilename = NULL;
+ ret->removeSrcFile = 0;
+ return ret;
+}
+
+void LZ4IO_freePreferences(LZ4IO_prefs_t* prefs)
+{
+ free(prefs);
+}
+
+
+int LZ4IO_setDictionaryFilename(LZ4IO_prefs_t* const prefs, const char* dictionaryFilename)
+{
+ prefs->dictionaryFilename = dictionaryFilename;
+ prefs->useDictionary = dictionaryFilename != NULL;
+ return prefs->useDictionary;
}
+/* Default setting : passThrough = 0; return : passThrough mode (0/1) */
+int LZ4IO_setPassThrough(LZ4IO_prefs_t* const prefs, int yes)
+{
+ prefs->passThrough = (yes!=0);
+ return prefs->passThrough;
+}
+
+
/* Default setting : overwrite = 1; return : overwrite mode (0/1) */
-int LZ4IO_setOverwrite(int yes)
+int LZ4IO_setOverwrite(LZ4IO_prefs_t* const prefs, int yes)
{
- g_overwrite = (yes!=0);
- return g_overwrite;
+ prefs->overwrite = (yes!=0);
+ return prefs->overwrite;
}
/* Default setting : testMode = 0; return : testMode (0/1) */
-int LZ4IO_setTestMode(int yes)
+int LZ4IO_setTestMode(LZ4IO_prefs_t* const prefs, int yes)
{
- g_testMode = (yes!=0);
- return g_testMode;
+ prefs->testMode = (yes!=0);
+ return prefs->testMode;
}
/* blockSizeID : valid values : 4-5-6-7 */
-size_t LZ4IO_setBlockSizeID(unsigned bsid)
+size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned bsid)
{
static const size_t blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB };
static const unsigned minBlockSizeID = 4;
static const unsigned maxBlockSizeID = 7;
if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0;
- g_blockSizeId = bsid;
- return blockSizeTable[g_blockSizeId-minBlockSizeID];
+ prefs->blockSizeId = (int)bsid;
+ prefs->blockSize = blockSizeTable[(unsigned)prefs->blockSizeId-minBlockSizeID];
+ return prefs->blockSize;
}
-int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode)
+size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize)
{
- g_blockIndependence = (blockMode == LZ4IO_blockIndependent);
- return g_blockIndependence;
+ static const size_t minBlockSize = 32;
+ static const size_t maxBlockSize = 4 MB;
+ unsigned bsid = 0;
+ if (blockSize < minBlockSize) blockSize = minBlockSize;
+ if (blockSize > maxBlockSize) blockSize = maxBlockSize;
+ prefs->blockSize = blockSize;
+ blockSize--;
+ /* find which of { 64k, 256k, 1MB, 4MB } is closest to blockSize */
+ while (blockSize >>= 2)
+ bsid++;
+ if (bsid < 7) bsid = 7;
+ prefs->blockSizeId = (int)(bsid-3);
+ return prefs->blockSize;
}
-/* Default setting : no block checksum */
-int LZ4IO_setBlockChecksumMode(int enable)
+/* Default setting : 1 == independent blocks */
+int LZ4IO_setBlockMode(LZ4IO_prefs_t* const prefs, LZ4IO_blockMode_t blockMode)
{
- g_blockChecksum = (enable != 0);
- return g_blockChecksum;
+ prefs->blockIndependence = (blockMode == LZ4IO_blockIndependent);
+ return prefs->blockIndependence;
}
-/* Default setting : checksum enabled */
-int LZ4IO_setStreamChecksumMode(int enable)
+/* Default setting : 0 == no block checksum */
+int LZ4IO_setBlockChecksumMode(LZ4IO_prefs_t* const prefs, int enable)
{
- g_streamChecksum = (enable != 0);
- return g_streamChecksum;
+ prefs->blockChecksum = (enable != 0);
+ return prefs->blockChecksum;
+}
+
+/* Default setting : 1 == checksum enabled */
+int LZ4IO_setStreamChecksumMode(LZ4IO_prefs_t* const prefs, int enable)
+{
+ prefs->streamChecksum = (enable != 0);
+ return prefs->streamChecksum;
}
/* Default setting : 0 (no notification) */
@@ -208,28 +270,30 @@ int LZ4IO_setNotificationLevel(int level)
return g_displayLevel;
}
-/* Default setting : 0 (disabled) */
-int LZ4IO_setSparseFile(int enable)
+/* Default setting : 1 (auto: enabled on file, disabled on stdout) */
+int LZ4IO_setSparseFile(LZ4IO_prefs_t* const prefs, int enable)
{
- g_sparseFileSupport = (enable!=0);
- return g_sparseFileSupport;
+ prefs->sparseFileSupport = 2*(enable!=0); /* 2==force enable */
+ return prefs->sparseFileSupport;
}
/* Default setting : 0 (disabled) */
-int LZ4IO_setContentSize(int enable)
+int LZ4IO_setContentSize(LZ4IO_prefs_t* const prefs, int enable)
{
- g_contentSizeFlag = (enable!=0);
- return g_contentSizeFlag;
+ prefs->contentSizeFlag = (enable!=0);
+ return prefs->contentSizeFlag;
}
/* Default setting : 0 (disabled) */
-void LZ4IO_favorDecSpeed(int favor)
+void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor)
{
- g_favorDecSpeed = (favor!=0);
+ prefs->favorDecSpeed = (favor!=0);
}
-static U32 g_removeSrcFile = 0;
-void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
+void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag)
+{
+ prefs->removeSrcFile = (flag>0);
+}
@@ -237,7 +301,6 @@ void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
** ********************** LZ4 File / Pipe compression ********************* **
** ************************************************************************ */
-static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
static int LZ4IO_isSkippableMagicNumber(unsigned int magic) {
return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0;
}
@@ -263,25 +326,27 @@ static FILE* LZ4IO_openSrcFile(const char* srcFileName)
}
/** FIO_openDstFile() :
- * condition : `dstFileName` must be non-NULL.
+ * prefs is writable, because sparseFileSupport might be updated.
+ * condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
-static FILE* LZ4IO_openDstFile(const char* dstFileName)
+static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* const prefs)
{
FILE* f;
+ assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) {
- DISPLAYLEVEL(4,"Using stdout for output\n");
+ DISPLAYLEVEL(4, "Using stdout for output \n");
f = stdout;
SET_BINARY_MODE(stdout);
- if (g_sparseFileSupport==1) {
- g_sparseFileSupport = 0;
- DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
+ if (prefs->sparseFileSupport==1) {
+ DISPLAYLEVEL(4, "Sparse File Support automatically disabled on stdout ;"
+ " to force-enable it, add --sparse command \n");
}
} else {
- if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
- f = fopen( dstFileName, "rb" );
- if (f != NULL) { /* dest exists, prompt for overwrite authorization */
- fclose(f);
+ if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
+ FILE* const testf = fopen( dstFileName, "rb" );
+ if (testf != NULL) { /* dest exists, prompt for overwrite authorization */
+ fclose(testf);
if (g_displayLevel <= 1) { /* No interaction possible */
DISPLAY("%s already exists; not overwritten \n", dstFileName);
return NULL;
@@ -299,7 +364,9 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName)
}
/* sparse file */
- if (f && g_sparseFileSupport) { SET_SPARSE_FILE_MODE(f); }
+ { int const sparseMode = (prefs->sparseFileSupport - (f==stdout)) > 0;
+ if (f && sparseMode) { SET_SPARSE_FILE_MODE(f); }
+ }
return f;
}
@@ -329,57 +396,64 @@ static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSi
/* LZ4IO_compressFilename_Legacy :
* This function is intentionally "hidden" (not published in .h)
* It generates compressed streams using the old 'legacy' format */
-int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel)
+int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename,
+ int compressionlevel, const LZ4IO_prefs_t* prefs)
{
- int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
+ typedef int (*compress_f)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
+ compress_f const compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC;
unsigned long long filesize = 0;
unsigned long long compressedfilesize = MAGICNUMBER_SIZE;
char* in_buff;
char* out_buff;
const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE);
- FILE* finput;
+ FILE* const finput = LZ4IO_openSrcFile(input_filename);
FILE* foutput;
clock_t clockEnd;
/* Init */
clock_t const clockStart = clock();
- compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC;
+ if (finput == NULL)
+ EXM_THROW(20, "%s : open file error ", input_filename);
- finput = LZ4IO_openSrcFile(input_filename);
- if (finput == NULL) EXM_THROW(20, "%s : open file error ", input_filename);
- foutput = LZ4IO_openDstFile(output_filename);
- if (foutput == NULL) { fclose(finput); EXM_THROW(20, "%s : open file error ", input_filename); }
+ foutput = LZ4IO_openDstFile(output_filename, prefs);
+ if (foutput == NULL) {
+ fclose(finput);
+ EXM_THROW(20, "%s : open file error ", input_filename);
+ }
/* Allocate Memory */
in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
- out_buff = (char*)malloc(outBuffSize);
- if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
+ out_buff = (char*)malloc((size_t)outBuffSize + 4);
+ if (!in_buff || !out_buff)
+ EXM_THROW(21, "Allocation error : not enough memory");
/* Write Archive Header */
LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER);
- { size_t const sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);
- if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); }
+ if (fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE)
+ EXM_THROW(22, "Write error : cannot write header");
/* Main Loop */
while (1) {
- unsigned int outSize;
+ int outSize;
/* Read Block */
- size_t const inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);
+ size_t const inSize = fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);
if (inSize == 0) break;
- if (inSize > LEGACY_BLOCKSIZE) EXM_THROW(23, "Read error : wrong fread() size report "); /* should be impossible */
+ assert(inSize <= LEGACY_BLOCKSIZE);
filesize += inSize;
/* Compress Block */
outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel);
- compressedfilesize += outSize+4;
+ assert(outSize >= 0);
+ compressedfilesize += (unsigned long long)outSize+4;
DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ",
(int)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
- LZ4IO_writeLE32(out_buff, outSize);
- { size_t const sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);
- if (sizeCheck!=(size_t)(outSize+4))
- EXM_THROW(24, "Write error : cannot write compressed block");
+ assert(outSize > 0);
+ assert(outSize < outBuffSize);
+ LZ4IO_writeLE32(out_buff, (unsigned)outSize);
+ if (fwrite(out_buff, 1, (size_t)outSize+4, foutput) != (size_t)(outSize+4)) {
+ EXM_THROW(24, "Write error : cannot write compressed block");
} }
if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename);
@@ -399,11 +473,59 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
free(in_buff);
free(out_buff);
fclose(finput);
- fclose(foutput);
+ if (strcmp(output_filename,stdoutmark)) fclose(foutput); /* do not close stdout */
return 0;
}
+#define FNSPACE 30
+/* LZ4IO_compressMultipleFilenames_Legacy :
+ * This function is intentionally "hidden" (not published in .h)
+ * It generates multiple compressed streams using the old 'legacy' format */
+int LZ4IO_compressMultipleFilenames_Legacy(
+ const char** inFileNamesTable, int ifntSize,
+ const char* suffix,
+ int compressionLevel, const LZ4IO_prefs_t* prefs)
+{
+ int i;
+ int missed_files = 0;
+ char* dstFileName = (char*)malloc(FNSPACE);
+ size_t ofnSize = FNSPACE;
+ const size_t suffixSize = strlen(suffix);
+
+ if (dstFileName == NULL) return ifntSize; /* not enough memory */
+
+ /* loop on each file */
+ for (i=0; i<ifntSize; i++) {
+ size_t const ifnSize = strlen(inFileNamesTable[i]);
+ if (!strcmp(suffix, stdoutmark)) {
+ missed_files += LZ4IO_compressFilename_Legacy(
+ inFileNamesTable[i], stdoutmark,
+ compressionLevel, prefs);
+ continue;
+ }
+
+ if (ofnSize <= ifnSize+suffixSize+1) {
+ free(dstFileName);
+ ofnSize = ifnSize + 20;
+ dstFileName = (char*)malloc(ofnSize);
+ if (dstFileName==NULL) {
+ return ifntSize;
+ } }
+ strcpy(dstFileName, inFileNamesTable[i]);
+ strcat(dstFileName, suffix);
+
+ missed_files += LZ4IO_compressFilename_Legacy(
+ inFileNamesTable[i], dstFileName,
+ compressionLevel, prefs);
+ }
+
+ /* Close & Free */
+ free(dstFileName);
+
+ return missed_files;
+}
+
/*********************************************
* Compression using Frame format
@@ -418,21 +540,20 @@ typedef struct {
LZ4F_CDict* cdict;
} cRess_t;
-static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) {
+static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename)
+{
size_t readSize;
size_t dictEnd = 0;
size_t dictLen = 0;
size_t dictStart;
size_t circularBufSize = LZ4_MAX_DICT_SIZE;
- char* circularBuf;
- char* dictBuf;
+ char* circularBuf = (char*)malloc(circularBufSize);
+ char* dictBuf;
FILE* dictFile;
+ if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory for circular buffer");
if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided");
- circularBuf = (char *) malloc(circularBufSize);
- if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory");
-
dictFile = LZ4IO_openSrcFile(dictFilename);
if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
@@ -462,7 +583,7 @@ static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) {
circularBuf = NULL;
} else {
/* Otherwise, we will alloc a new buffer and copy our dict into that. */
- dictBuf = (char *) malloc(dictLen ? dictLen : 1);
+ dictBuf = (char *)malloc(dictLen ? dictLen : 1);
if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart);
@@ -475,23 +596,22 @@ static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) {
return dictBuf;
}
-static LZ4F_CDict* LZ4IO_createCDict(void) {
+static LZ4F_CDict* LZ4IO_createCDict(const LZ4IO_prefs_t* const prefs)
+{
size_t dictionarySize;
void* dictionaryBuffer;
LZ4F_CDict* cdict;
- if (!g_useDictionary) {
- return NULL;
- }
- dictionaryBuffer = LZ4IO_createDict(g_dictionaryFilename, &dictionarySize);
+ if (!prefs->useDictionary) return NULL;
+ dictionaryBuffer = LZ4IO_createDict(&dictionarySize, prefs->dictionaryFilename);
if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize);
free(dictionaryBuffer);
return cdict;
}
-static cRess_t LZ4IO_createCResources(void)
+static cRess_t LZ4IO_createCResources(const LZ4IO_prefs_t* const prefs)
{
- const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
+ const size_t blockSize = prefs->blockSize;
cRess_t ress;
LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION);
@@ -504,7 +624,7 @@ static cRess_t LZ4IO_createCResources(void)
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
- ress.cdict = LZ4IO_createCDict();
+ ress.cdict = LZ4IO_createCDict(prefs);
return ress;
}
@@ -526,38 +646,39 @@ static void LZ4IO_freeCResources(cRess_t ress)
* result : 0 : compression completed correctly
* 1 : missing or pb opening srcFileName
*/
-static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel)
+static int
+LZ4IO_compressFilename_extRess(cRess_t ress,
+ const char* srcFileName, const char* dstFileName,
+ int compressionLevel, const LZ4IO_prefs_t* const io_prefs)
{
unsigned long long filesize = 0;
unsigned long long compressedfilesize = 0;
- FILE* srcFile;
FILE* dstFile;
void* const srcBuffer = ress.srcBuffer;
void* const dstBuffer = ress.dstBuffer;
const size_t dstBufferSize = ress.dstBufferSize;
- const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
+ const size_t blockSize = io_prefs->blockSize;
size_t readSize;
LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */
LZ4F_preferences_t prefs;
/* Init */
- srcFile = LZ4IO_openSrcFile(srcFileName);
+ FILE* const srcFile = LZ4IO_openSrcFile(srcFileName);
if (srcFile == NULL) return 1;
- dstFile = LZ4IO_openDstFile(dstFileName);
+ dstFile = LZ4IO_openDstFile(dstFileName, io_prefs);
if (dstFile == NULL) { fclose(srcFile); return 1; }
memset(&prefs, 0, sizeof(prefs));
-
/* Set compression parameters */
prefs.autoFlush = 1;
prefs.compressionLevel = compressionLevel;
- prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence;
- prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId;
- prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum;
- prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum;
- prefs.favorDecSpeed = g_favorDecSpeed;
- if (g_contentSizeFlag) {
- U64 const fileSize = UTIL_getFileSize(srcFileName);
+ prefs.frameInfo.blockMode = (LZ4F_blockMode_t)io_prefs->blockIndependence;
+ prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)io_prefs->blockSizeId;
+ prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)io_prefs->blockChecksum;
+ prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)io_prefs->streamChecksum;
+ prefs.favorDecSpeed = io_prefs->favorDecSpeed;
+ if (io_prefs->contentSizeFlag) {
+ U64 const fileSize = UTIL_getOpenFileSize(srcFile);
prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */
if (fileSize==0)
DISPLAYLEVEL(3, "Warning : cannot determine input content size \n");
@@ -571,41 +692,41 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
/* single-block file */
if (readSize < blockSize) {
/* Compress in single pass */
- size_t cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
- if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
+ size_t const cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
+ if (LZ4F_isError(cSize))
+ EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
compressedfilesize = cSize;
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
(unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */
/* Write Block */
- { size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile);
- if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block");
+ if (fwrite(dstBuffer, 1, cSize, dstFile) != cSize) {
+ EXM_THROW(32, "Write error : failed writing single-block compressed frame");
} }
else
/* multiple-blocks file */
{
- /* Write Archive Header */
- size_t headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
+ /* Write Frame Header */
+ size_t const headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
- { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
- if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); }
+ if (fwrite(dstBuffer, 1, headerSize, dstFile) != headerSize)
+ EXM_THROW(34, "Write error : cannot write header");
compressedfilesize += headerSize;
- /* Main Loop */
+ /* Main Loop - one block at a time */
while (readSize>0) {
- size_t outSize;
-
- /* Compress Block */
- outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
- if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
+ size_t const outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
+ if (LZ4F_isError(outSize))
+ EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
compressedfilesize += outSize;
- DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
+ DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
+ (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
- { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile);
- if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); }
+ if (fwrite(dstBuffer, 1, outSize, dstFile) != outSize)
+ EXM_THROW(36, "Write error : cannot write compressed block");
/* Read next block */
readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
@@ -613,18 +734,18 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
}
if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName);
- /* End of Stream mark */
- headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
- if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
-
- { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
- if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); }
- compressedfilesize += headerSize;
- }
+ /* End of Frame mark */
+ { size_t const endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
+ if (LZ4F_isError(endSize))
+ EXM_THROW(38, "End of frame error : %s", LZ4F_getErrorName(endSize));
+ if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize)
+ EXM_THROW(39, "Write error : cannot write end of frame");
+ compressedfilesize += endSize;
+ } }
- /* Release files */
+ /* Release file handlers */
fclose (srcFile);
- fclose (dstFile);
+ if (strcmp(dstFileName,stdoutmark)) fclose (dstFile); /* do not close stdout */
/* Copy owner, file permissions and modification time */
{ stat_t statbuf;
@@ -635,7 +756,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
UTIL_setFileStat(dstFileName, &statbuf);
} }
- if (g_removeSrcFile) { /* remove source file : --rm */
+ if (io_prefs->removeSrcFile) { /* remove source file : --rm */
if (remove(srcFileName))
EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno));
}
@@ -650,13 +771,13 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
}
-int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel)
+int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel, const LZ4IO_prefs_t* prefs)
{
UTIL_time_t const timeStart = UTIL_getTime();
clock_t const cpuStart = clock();
- cRess_t const ress = LZ4IO_createCResources();
+ cRess_t const ress = LZ4IO_createCResources(prefs);
- int const result = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel);
+ int const result = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel, prefs);
/* Free resources */
LZ4IO_freeCResources(ress);
@@ -674,8 +795,11 @@ int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int
}
-#define FNSPACE 30
-int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel)
+int LZ4IO_compressMultipleFilenames(
+ const char** inFileNamesTable, int ifntSize,
+ const char* suffix,
+ int compressionLevel,
+ const LZ4IO_prefs_t* prefs)
{
int i;
int missed_files = 0;
@@ -685,16 +809,31 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
cRess_t ress;
if (dstFileName == NULL) return ifntSize; /* not enough memory */
- ress = LZ4IO_createCResources();
+ ress = LZ4IO_createCResources(prefs);
/* loop on each file */
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
- if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); if (dstFileName==NULL) { LZ4IO_freeCResources(ress); return ifntSize; } }
+ if (!strcmp(suffix, stdoutmark)) {
+ missed_files += LZ4IO_compressFilename_extRess(ress,
+ inFileNamesTable[i], stdoutmark,
+ compressionLevel, prefs);
+ continue;
+ }
+ if (ofnSize <= ifnSize+suffixSize+1) {
+ free(dstFileName);
+ ofnSize = ifnSize + 20;
+ dstFileName = (char*)malloc(ofnSize);
+ if (dstFileName==NULL) {
+ LZ4IO_freeCResources(ress);
+ return ifntSize;
+ } }
strcpy(dstFileName, inFileNamesTable[i]);
strcat(dstFileName, suffix);
- missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel);
+ missed_files += LZ4IO_compressFilename_extRess(ress,
+ inFileNamesTable[i], dstFileName,
+ compressionLevel, prefs);
}
/* Close & Free */
@@ -709,18 +848,23 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
/* ********************** LZ4 file-stream Decompression **************** */
/* ********************************************************************* */
+/* It's presumed that s points to a memory space of size >= 4 */
static unsigned LZ4IO_readLE32 (const void* s)
{
const unsigned char* const srcPtr = (const unsigned char*)s;
unsigned value32 = srcPtr[0];
- value32 += (srcPtr[1]<<8);
- value32 += (srcPtr[2]<<16);
- value32 += ((unsigned)srcPtr[3])<<24;
+ value32 += (unsigned)srcPtr[1] << 8;
+ value32 += (unsigned)srcPtr[2] << 16;
+ value32 += (unsigned)srcPtr[3] << 24;
return value32;
}
-static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
+static unsigned
+LZ4IO_fwriteSparse(FILE* file,
+ const void* buffer, size_t bufferSize,
+ int sparseFileSupport,
+ unsigned storedSkips)
{
const size_t sizeT = sizeof(size_t);
const size_t maskT = sizeT -1 ;
@@ -729,8 +873,9 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
size_t bufferSizeT = bufferSize / sizeT;
const size_t* const bufferTEnd = bufferT + bufferSizeT;
const size_t segmentSizeT = (32 KB) / sizeT;
+ int const sparseMode = (sparseFileSupport - (file==stdout)) > 0;
- if (!g_sparseFileSupport) { /* normal write */
+ if (!sparseMode) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
return 0;
@@ -778,7 +923,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
storedSkips = 0;
- { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
+ { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
} }
}
@@ -788,24 +933,24 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
{
- if (storedSkips>0) { /* implies g_sparseFileSupport>0 */
- int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR);
- if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
- { const char lastZeroByte[1] = { 0 };
- size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
- if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
- } }
+ if (storedSkips>0) { /* implies sparseFileSupport>0 */
+ const char lastZeroByte[1] = { 0 };
+ if (UTIL_fseek(file, storedSkips-1, SEEK_CUR) != 0)
+ EXM_THROW(69, "Final skip error (sparse file)\n");
+ if (fwrite(lastZeroByte, 1, 1, file) != 1)
+ EXM_THROW(69, "Write error : cannot write last zero\n");
+ }
}
static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */
-static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
+static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs)
{
unsigned long long streamSize = 0;
unsigned storedSkips = 0;
/* Allocate Memory */
- char* const in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
+ char* const in_buff = (char*)malloc((size_t)LZ4_compressBound(LEGACY_BLOCKSIZE));
char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
@@ -829,11 +974,11 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); }
/* Decode Block */
- { int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
+ { int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, (int)blockSize, LEGACY_BLOCKSIZE);
if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
- streamSize += decodeSize;
+ streamSize += (unsigned long long)decodeSize;
/* Write Block */
- storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); /* success or die */
+ storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, (size_t)decodeSize, prefs->sparseFileSupport, storedSkips); /* success or die */
} }
if (ferror(finput)) EXM_THROW(54, "Read error : ferror");
@@ -859,19 +1004,20 @@ typedef struct {
size_t dictBufferSize;
} dRess_t;
-static void LZ4IO_loadDDict(dRess_t* ress) {
- if (!g_useDictionary) {
+static void LZ4IO_loadDDict(dRess_t* ress, const LZ4IO_prefs_t* const prefs)
+{
+ if (!prefs->useDictionary) {
ress->dictBuffer = NULL;
ress->dictBufferSize = 0;
return;
}
- ress->dictBuffer = LZ4IO_createDict(g_dictionaryFilename, &ress->dictBufferSize);
+ ress->dictBuffer = LZ4IO_createDict(&ress->dictBufferSize, prefs->dictionaryFilename);
if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
}
static const size_t LZ4IO_dBufferSize = 64 KB;
-static dRess_t LZ4IO_createDResources(void)
+static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs)
{
dRess_t ress;
@@ -886,7 +1032,7 @@ static dRess_t LZ4IO_createDResources(void)
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
- LZ4IO_loadDDict(&ress);
+ LZ4IO_loadDDict(&ress, prefs);
ress.dstFile = NULL;
return ress;
@@ -902,7 +1048,10 @@ static void LZ4IO_freeDResources(dRess_t ress)
}
-static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile)
+static unsigned long long
+LZ4IO_decompressLZ4F(dRess_t ress,
+ FILE* const srcFile, FILE* const dstFile,
+ const LZ4IO_prefs_t* const prefs)
{
unsigned long long filesize = 0;
LZ4F_errorCode_t nextToLoad;
@@ -937,8 +1086,8 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
/* Write Block */
if (decodedBytes) {
- if (!g_testMode)
- storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips);
+ if (!prefs->testMode)
+ storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, prefs->sparseFileSupport, storedSkips);
filesize += decodedBytes;
DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
}
@@ -949,29 +1098,37 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
/* can be out because readSize == 0, which could be an fread() error */
if (ferror(srcFile)) EXM_THROW(67, "Read error");
- if (!g_testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips);
+ if (!prefs->testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips);
if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream");
return filesize;
}
+/* LZ4IO_passThrough:
+ * just output the same content as input, no decoding.
+ * This is a capability of zcat, and by extension lz4cat
+ * MNstore : contain the first MAGICNUMBER_SIZE bytes already read from finput
+ */
#define PTSIZE (64 KB)
#define PTSIZET (PTSIZE / sizeof(size_t))
-static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE])
+static unsigned long long
+LZ4IO_passThrough(FILE* finput, FILE* foutput,
+ unsigned char MNstore[MAGICNUMBER_SIZE],
+ int sparseFileSupport)
{
size_t buffer[PTSIZET];
size_t readBytes = 1;
unsigned long long total = MAGICNUMBER_SIZE;
unsigned storedSkips = 0;
- size_t const sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput);
- if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error");
-
+ if (fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) {
+ EXM_THROW(50, "Pass-through write error");
+ }
while (readBytes) {
- readBytes = fread(buffer, 1, PTSIZE, finput);
+ readBytes = fread(buffer, 1, sizeof(buffer), finput);
total += readBytes;
- storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, storedSkips);
+ storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, sparseFileSupport, storedSkips);
}
if (ferror(finput)) EXM_THROW(51, "Read Error");
@@ -998,7 +1155,10 @@ static int fseek_u32(FILE *fp, unsigned offset, int where)
}
#define ENDOFSTREAM ((unsigned long long)-1)
-static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput)
+static unsigned long long
+selectDecoder(dRess_t ress,
+ FILE* finput, FILE* foutput,
+ const LZ4IO_prefs_t* const prefs)
{
unsigned char MNstore[MAGICNUMBER_SIZE];
unsigned magicNumber;
@@ -1024,10 +1184,10 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
switch(magicNumber)
{
case LZ4IO_MAGICNUMBER:
- return LZ4IO_decompressLZ4F(ress, finput, foutput);
+ return LZ4IO_decompressLZ4F(ress, finput, foutput, prefs);
case LEGACY_MAGICNUMBER:
DISPLAYLEVEL(4, "Detected : Legacy format \n");
- return LZ4IO_decodeLegacyStream(finput, foutput);
+ return LZ4IO_decodeLegacyStream(finput, foutput, prefs);
case LZ4IO_SKIPPABLE0:
DISPLAYLEVEL(4, "Skipping detected skippable area \n");
{ size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
@@ -1044,9 +1204,9 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
default:
if (nbFrames == 1) { /* just started */
/* Wrong magic number at the beginning of 1st stream */
- if (!g_testMode && g_overwrite) {
+ if (!prefs->testMode && prefs->overwrite && prefs->passThrough) {
nbFrames = 0;
- return LZ4IO_passThrough(finput, foutput, MNstore);
+ return LZ4IO_passThrough(finput, foutput, MNstore, prefs->sparseFileSupport);
}
EXM_THROW(44,"Unrecognized header : file cannot be decoded");
}
@@ -1061,7 +1221,10 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
}
-static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename)
+static int
+LZ4IO_decompressSrcFile(dRess_t ress,
+ const char* input_filename, const char* output_filename,
+ const LZ4IO_prefs_t* const prefs)
{
FILE* const foutput = ress.dstFile;
unsigned long long filesize = 0;
@@ -1069,18 +1232,19 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con
/* Init */
FILE* const finput = LZ4IO_openSrcFile(input_filename);
if (finput==NULL) return 1;
+ assert(foutput != NULL);
/* Loop over multiple streams */
for ( ; ; ) { /* endless loop, see break condition */
unsigned long long const decodedSize =
- selectDecoder(ress, finput, foutput);
+ selectDecoder(ress, finput, foutput, prefs);
if (decodedSize == ENDOFSTREAM) break;
filesize += decodedSize;
}
/* Close input */
fclose(finput);
- if (g_removeSrcFile) { /* --rm */
+ if (prefs->removeSrcFile) { /* --rm */
if (remove(input_filename))
EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno));
}
@@ -1094,11 +1258,14 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con
}
-static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename)
+static int
+LZ4IO_decompressDstFile(dRess_t ress,
+ const char* input_filename, const char* output_filename,
+ const LZ4IO_prefs_t* const prefs)
{
stat_t statbuf;
int stat_result = 0;
- FILE* const foutput = LZ4IO_openDstFile(output_filename);
+ FILE* const foutput = LZ4IO_openDstFile(output_filename, prefs);
if (foutput==NULL) return 1; /* failure */
if ( strcmp(input_filename, stdinmark)
@@ -1106,7 +1273,7 @@ static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, con
stat_result = 1;
ress.dstFile = foutput;
- LZ4IO_decompressSrcFile(ress, input_filename, output_filename);
+ LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs);
fclose(foutput);
@@ -1122,12 +1289,12 @@ static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, con
}
-int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename)
+int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename, const LZ4IO_prefs_t* prefs)
{
- dRess_t const ress = LZ4IO_createDResources();
+ dRess_t const ress = LZ4IO_createDResources(prefs);
clock_t const start = clock();
- int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename);
+ int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename, prefs);
clock_t const end = clock();
double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
@@ -1138,7 +1305,10 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file
}
-int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix)
+int LZ4IO_decompressMultipleFilenames(
+ const char** inFileNamesTable, int ifntSize,
+ const char* suffix,
+ const LZ4IO_prefs_t* prefs)
{
int i;
int skippedFiles = 0;
@@ -1146,19 +1316,24 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz
char* outFileName = (char*)malloc(FNSPACE);
size_t ofnSize = FNSPACE;
size_t const suffixSize = strlen(suffix);
- dRess_t ress = LZ4IO_createDResources();
+ dRess_t ress = LZ4IO_createDResources(prefs);
- if (outFileName==NULL) return ifntSize; /* not enough memory */
- ress.dstFile = LZ4IO_openDstFile(stdoutmark);
+ if (outFileName==NULL) EXM_THROW(70, "Memory allocation error");
+ ress.dstFile = LZ4IO_openDstFile(stdoutmark, prefs);
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize;
if (!strcmp(suffix, stdoutmark)) {
- missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark);
+ missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark, prefs);
continue;
}
- if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) return ifntSize; }
+ if (ofnSize <= ifnSize-suffixSize+1) {
+ free(outFileName);
+ ofnSize = ifnSize + 20;
+ outFileName = (char*)malloc(ofnSize);
+ if (outFileName==NULL) EXM_THROW(71, "Memory allocation error");
+ }
if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
skippedFiles++;
@@ -1166,10 +1341,337 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz
}
memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize);
outFileName[ifnSize-suffixSize] = '\0';
- missingFiles += LZ4IO_decompressDstFile(ress, inFileNamesTable[i], outFileName);
+ missingFiles += LZ4IO_decompressDstFile(ress, inFileNamesTable[i], outFileName, prefs);
}
LZ4IO_freeDResources(ress);
free(outFileName);
return missingFiles + skippedFiles;
}
+
+
+/* ********************************************************************* */
+/* ********************** LZ4 --list command *********************** */
+/* ********************************************************************* */
+
+typedef enum
+{
+ lz4Frame = 0,
+ legacyFrame,
+ skippableFrame
+} LZ4IO_frameType_t;
+
+typedef struct {
+ LZ4F_frameInfo_t lz4FrameInfo;
+ LZ4IO_frameType_t frameType;
+} LZ4IO_frameInfo_t;
+
+#define LZ4IO_INIT_FRAMEINFO { LZ4F_INIT_FRAMEINFO, lz4Frame }
+
+typedef struct {
+ const char* fileName;
+ unsigned long long fileSize;
+ unsigned long long frameCount;
+ LZ4IO_frameInfo_t frameSummary;
+ unsigned short eqFrameTypes;
+ unsigned short eqBlockTypes;
+ unsigned short allContentSize;
+} LZ4IO_cFileInfo_t;
+
+#define LZ4IO_INIT_CFILEINFO { NULL, 0ULL, 0, LZ4IO_INIT_FRAMEINFO, 1, 1, 1 }
+
+typedef enum { LZ4IO_LZ4F_OK, LZ4IO_format_not_known, LZ4IO_not_a_file } LZ4IO_infoResult;
+
+static const char * LZ4IO_frameTypeNames[] = {"LZ4Frame", "LegacyFrame", "SkippableFrame" };
+
+/* Read block headers and skip block data
+ Return total blocks size for this frame including block headers,
+ block checksums and content checksums.
+ returns 0 in case it can't succesfully skip block data.
+ Assumes SEEK_CUR after frame header.
+ */
+static unsigned long long
+LZ4IO_skipBlocksData(FILE* finput,
+ const LZ4F_blockChecksum_t blockChecksumFlag,
+ const LZ4F_contentChecksum_t contentChecksumFlag)
+{
+ unsigned char blockInfo[LZ4F_BLOCK_HEADER_SIZE];
+ unsigned long long totalBlocksSize = 0;
+ for (;;) {
+ if (!fread(blockInfo, 1, LZ4F_BLOCK_HEADER_SIZE, finput)) {
+ if (feof(finput)) return totalBlocksSize;
+ return 0;
+ }
+ totalBlocksSize += LZ4F_BLOCK_HEADER_SIZE;
+ { const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU;
+ const unsigned long nextBlock = nextCBlockSize + (blockChecksumFlag * LZ4F_BLOCK_CHECKSUM_SIZE);
+ if (nextCBlockSize == 0) {
+ /* Reached EndMark */
+ if (contentChecksumFlag) {
+ /* Skip content checksum */
+ if (UTIL_fseek(finput, LZ4F_CONTENT_CHECKSUM_SIZE, SEEK_CUR) != 0) {
+ return 0;
+ }
+ totalBlocksSize += LZ4F_CONTENT_CHECKSUM_SIZE;
+ }
+ break;
+ }
+ totalBlocksSize += nextBlock;
+ /* skip to the next block */
+ assert(nextBlock < LONG_MAX);
+ if (UTIL_fseek(finput, (long)nextBlock, SEEK_CUR) != 0) return 0;
+ } }
+ return totalBlocksSize;
+}
+
+/* For legacy frames only.
+ Read block headers and skip block data.
+ Return total blocks size for this frame including block headers.
+ or 0 in case it can't succesfully skip block data.
+ This works as long as legacy block header size = magic number size.
+ Assumes SEEK_CUR after frame header.
+ */
+static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput)
+{
+ unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE];
+ unsigned long long totalBlocksSize = 0;
+ LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE);
+ for (;;) {
+ if (!fread(blockInfo, 1, LZIO_LEGACY_BLOCK_HEADER_SIZE, finput)) {
+ if (feof(finput)) return totalBlocksSize;
+ return 0;
+ }
+ { const unsigned int nextCBlockSize = LZ4IO_readLE32(&blockInfo);
+ if ( nextCBlockSize == LEGACY_MAGICNUMBER ||
+ nextCBlockSize == LZ4IO_MAGICNUMBER ||
+ LZ4IO_isSkippableMagicNumber(nextCBlockSize)) {
+ /* Rewind back. we want cursor at the begining of next frame.*/
+ if (fseek(finput, -LZIO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) {
+ return 0;
+ }
+ break;
+ }
+ totalBlocksSize += LZIO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize;
+ /* skip to the next block */
+ if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) {
+ return 0;
+ } } }
+ return totalBlocksSize;
+}
+
+/* LZ4IO_blockTypeID:
+ * return human-readable block type, following command line convention
+ * buffer : must be a valid memory area of at least 4 bytes */
+const char* LZ4IO_blockTypeID(LZ4F_blockSizeID_t sizeID, LZ4F_blockMode_t blockMode, char buffer[4])
+{
+ buffer[0] = 'B';
+ assert(sizeID >= 4); assert(sizeID <= 7);
+ buffer[1] = (char)(sizeID + '0');
+ buffer[2] = (blockMode == LZ4F_blockIndependent) ? 'I' : 'D';
+ buffer[3] = 0;
+ return buffer;
+}
+
+/* buffer : must be valid memory area of at least 10 bytes */
+static const char* LZ4IO_toHuman(long double size, char *buf)
+{
+ const char units[] = {"\0KMGTPEZY"};
+ size_t i = 0;
+ for (; size >= 1024; i++) size /= 1024;
+ sprintf(buf, "%.2Lf%c", size, units[i]);
+ return buf;
+}
+
+/* Get filename without path prefix */
+static const char* LZ4IO_baseName(const char* input_filename)
+{
+ const char* b = strrchr(input_filename, '/');
+ if (!b) b = strrchr(input_filename, '\\');
+ if (!b) return input_filename;
+ return b + 1;
+}
+
+/* Report frame/s information (--list) in verbose mode (-v).
+ * Will populate file info with fileName and frameSummary where applicable.
+ * - TODO :
+ * + report nb of blocks, hence max. possible decompressed size (when not reported in header)
+ */
+static LZ4IO_infoResult
+LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename)
+{
+ LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */
+ unsigned char buffer[LZ4F_HEADER_SIZE_MAX];
+ FILE* const finput = LZ4IO_openSrcFile(input_filename);
+
+ if (finput == NULL) return LZ4IO_not_a_file;
+ cfinfo->fileSize = UTIL_getOpenFileSize(finput);
+
+ while (!feof(finput)) {
+ LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO;
+ unsigned magicNumber;
+ /* Get MagicNumber */
+ { size_t const nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput);
+ if (nbReadBytes == 0) { break; } /* EOF */
+ result = LZ4IO_format_not_known; /* default result (error) */
+ if (nbReadBytes != MAGICNUMBER_SIZE) {
+ EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+ } }
+ magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */
+ if (LZ4IO_isSkippableMagicNumber(magicNumber))
+ magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */
+
+ switch (magicNumber) {
+ case LZ4IO_MAGICNUMBER:
+ if (cfinfo->frameSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0;
+ /* Get frame info */
+ { const size_t readBytes = fread(buffer + MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN - MAGICNUMBER_SIZE, finput);
+ if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename);
+ }
+ { size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN);
+ if (LZ4F_isError(hSize)) break;
+ if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) {
+ /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/
+ const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput);
+ if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename);
+ }
+ /* Create decompression context */
+ { LZ4F_dctx* dctx;
+ if ( LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) break;
+ { unsigned const frameInfoError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize));
+ LZ4F_freeDecompressionContext(dctx);
+ if (frameInfoError) break;
+ if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID ||
+ cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode)
+ && cfinfo->frameCount != 0)
+ cfinfo->eqBlockTypes = 0;
+ { const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput,
+ frameInfo.lz4FrameInfo.blockChecksumFlag,
+ frameInfo.lz4FrameInfo.contentChecksumFlag);
+ if (totalBlocksSize) {
+ char bTypeBuffer[5];
+ LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer);
+ DISPLAYLEVEL(3, " %6llu %14s %5s %8s",
+ cfinfo->frameCount + 1,
+ LZ4IO_frameTypeNames[frameInfo.frameType],
+ bTypeBuffer,
+ frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-");
+ if (frameInfo.lz4FrameInfo.contentSize) {
+ { double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100;
+ DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n",
+ totalBlocksSize + hSize,
+ frameInfo.lz4FrameInfo.contentSize,
+ ratio);
+ }
+ /* Now we've consumed frameInfo we can use it to store the total contentSize */
+ frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize;
+ }
+ else {
+ DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-");
+ cfinfo->allContentSize = 0;
+ }
+ result = LZ4IO_LZ4F_OK;
+ } } } } }
+ break;
+ case LEGACY_MAGICNUMBER:
+ frameInfo.frameType = legacyFrame;
+ if (cfinfo->frameSummary.frameType != legacyFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0;
+ cfinfo->eqBlockTypes = 0;
+ cfinfo->allContentSize = 0;
+ { const unsigned long long totalBlocksSize = LZ4IO_skipLegacyBlocksData(finput);
+ if (totalBlocksSize) {
+ DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20llu %20s %9s\n",
+ cfinfo->frameCount + 1,
+ LZ4IO_frameTypeNames[frameInfo.frameType],
+ "-", "-",
+ totalBlocksSize + 4,
+ "-", "-");
+ result = LZ4IO_LZ4F_OK;
+ } }
+ break;
+ case LZ4IO_SKIPPABLE0:
+ frameInfo.frameType = skippableFrame;
+ if (cfinfo->frameSummary.frameType != skippableFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0;
+ cfinfo->eqBlockTypes = 0;
+ cfinfo->allContentSize = 0;
+ { size_t const nbReadBytes = fread(buffer, 1, 4, finput);
+ if (nbReadBytes != 4)
+ EXM_THROW(42, "Stream error : skippable size unreadable");
+ }
+ { unsigned const size = LZ4IO_readLE32(buffer);
+ int const errorNb = fseek_u32(finput, size, SEEK_CUR);
+ if (errorNb != 0)
+ EXM_THROW(43, "Stream error : cannot skip skippable area");
+ DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n",
+ cfinfo->frameCount + 1,
+ "SkippableFrame",
+ "-", "-", size + 8, "-", "-");
+
+ result = LZ4IO_LZ4F_OK;
+ }
+ break;
+ default:
+ { long int const position = ftell(finput); /* only works for files < 2 GB */
+ DISPLAYLEVEL(3, "Stream followed by undecodable data ");
+ if (position != -1L)
+ DISPLAYLEVEL(3, "at position %i ", (int)position);
+ DISPLAYLEVEL(3, "\n");
+ }
+ break;
+ }
+ if (result != LZ4IO_LZ4F_OK) break;
+ cfinfo->frameSummary = frameInfo;
+ cfinfo->frameCount++;
+ } /* while (!feof(finput)) */
+ fclose(finput);
+ return result;
+}
+
+
+int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx)
+{
+ int result = 0;
+ size_t idx = 0;
+ if (g_displayLevel < 3) {
+ DISPLAYOUT("%10s %14s %5s %11s %13s %9s %s\n",
+ "Frames", "Type", "Block", "Compressed", "Uncompressed", "Ratio", "Filename");
+ }
+ for (; idx < ifnIdx; idx++) {
+ /* Get file info */
+ LZ4IO_cFileInfo_t cfinfo = LZ4IO_INIT_CFILEINFO;
+ cfinfo.fileName = LZ4IO_baseName(inFileNames[idx]);
+ if (!UTIL_isRegFile(inFileNames[idx])) {
+ DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]);
+ return 0;
+ }
+ DISPLAYLEVEL(3, "%s(%llu/%llu)\n", cfinfo.fileName, (unsigned long long)idx + 1, (unsigned long long)ifnIdx);
+ DISPLAYLEVEL(3, " %6s %14s %5s %8s %20s %20s %9s\n",
+ "Frame", "Type", "Block", "Checksum", "Compressed", "Uncompressed", "Ratio")
+ { LZ4IO_infoResult const op_result = LZ4IO_getCompressedFileInfo(&cfinfo, inFileNames[idx]);
+ if (op_result != LZ4IO_LZ4F_OK) {
+ assert(op_result == LZ4IO_format_not_known);
+ DISPLAYLEVEL(1, "lz4: %s: File format not recognized \n", inFileNames[idx]);
+ return 0;
+ } }
+ DISPLAYLEVEL(3, "\n");
+ if (g_displayLevel < 3) {
+ /* Display Summary */
+ { char buffers[3][10];
+ DISPLAYOUT("%10llu %14s %5s %11s %13s ",
+ cfinfo.frameCount,
+ cfinfo.eqFrameTypes ? LZ4IO_frameTypeNames[cfinfo.frameSummary.frameType] : "-" ,
+ cfinfo.eqBlockTypes ? LZ4IO_blockTypeID(cfinfo.frameSummary.lz4FrameInfo.blockSizeID,
+ cfinfo.frameSummary.lz4FrameInfo.blockMode, buffers[0]) : "-",
+ LZ4IO_toHuman((long double)cfinfo.fileSize, buffers[1]),
+ cfinfo.allContentSize ? LZ4IO_toHuman((long double)cfinfo.frameSummary.lz4FrameInfo.contentSize, buffers[2]) : "-");
+ if (cfinfo.allContentSize) {
+ double const ratio = (double)cfinfo.fileSize / cfinfo.frameSummary.lz4FrameInfo.contentSize * 100;
+ DISPLAYOUT("%9.2f%% %s \n", ratio, cfinfo.fileName);
+ } else {
+ DISPLAYOUT("%9s %s\n",
+ "-",
+ cfinfo.fileName);
+ } } } /* if (g_displayLevel < 3) */
+ } /* for (; idx < ifnIdx; idx++) */
+
+ return result;
+}
diff --git a/programs/lz4io.h b/programs/lz4io.h
index 22c5e3e..d6d7eee 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -39,70 +39,96 @@
/* ************************************************** */
/* Special input/output values */
/* ************************************************** */
+#define stdinmark "stdin"
+#define stdoutmark "stdout"
#define NULL_OUTPUT "null"
-static const char stdinmark[] = "stdin";
-static const char stdoutmark[] = "stdout";
#ifdef _WIN32
-static const char nulmark[] = "nul";
+#define nulmark "nul"
#else
-static const char nulmark[] = "/dev/null";
+#define nulmark "/dev/null"
#endif
+/* ************************************************** */
+/* ****************** Type Definitions ************** */
+/* ************************************************** */
+
+typedef struct LZ4IO_prefs_s LZ4IO_prefs_t;
+
+LZ4IO_prefs_t* LZ4IO_defaultPreferences(void);
+void LZ4IO_freePreferences(LZ4IO_prefs_t* prefs);
+
+/* Size in bytes of a legacy block header in little-endian format */
+#define LZIO_LEGACY_BLOCK_HEADER_SIZE 4
/* ************************************************** */
/* ****************** Functions ********************* */
/* ************************************************** */
-int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel);
-int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename);
+/* if output_filename == stdoutmark, writes to stdout */
+int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionlevel, const LZ4IO_prefs_t* prefs);
+int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename, const LZ4IO_prefs_t* prefs);
-int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel);
-int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix);
+/* if suffix == stdoutmark, writes to stdout */
+int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel, const LZ4IO_prefs_t* prefs);
+int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, const LZ4IO_prefs_t* prefs);
/* ************************************************** */
/* ****************** Parameters ******************** */
/* ************************************************** */
-int LZ4IO_setDictionaryFilename(const char* dictionaryFilename);
+int LZ4IO_setDictionaryFilename(LZ4IO_prefs_t* const prefs, const char* dictionaryFilename);
+
+/* Default setting : passThrough = 0;
+ return : passThrough mode (0/1) */
+int LZ4IO_setPassThrough(LZ4IO_prefs_t* const prefs, int yes);
/* Default setting : overwrite = 1;
return : overwrite mode (0/1) */
-int LZ4IO_setOverwrite(int yes);
+int LZ4IO_setOverwrite(LZ4IO_prefs_t* const prefs, int yes);
/* Default setting : testMode = 0;
return : testMode (0/1) */
-int LZ4IO_setTestMode(int yes);
+int LZ4IO_setTestMode(LZ4IO_prefs_t* const prefs, int yes);
/* blockSizeID : valid values : 4-5-6-7
return : 0 if error, blockSize if OK */
-size_t LZ4IO_setBlockSizeID(unsigned blockSizeID);
+size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned blockSizeID);
+
+/* blockSize : valid values : 32 -> 4MB
+ return : 0 if error, actual blocksize if OK */
+size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize);
/* Default setting : independent blocks */
typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t;
-int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode);
+int LZ4IO_setBlockMode(LZ4IO_prefs_t* const prefs, LZ4IO_blockMode_t blockMode);
/* Default setting : no block checksum */
-int LZ4IO_setBlockChecksumMode(int xxhash);
+int LZ4IO_setBlockChecksumMode(LZ4IO_prefs_t* const prefs, int xxhash);
/* Default setting : stream checksum enabled */
-int LZ4IO_setStreamChecksumMode(int xxhash);
+int LZ4IO_setStreamChecksumMode(LZ4IO_prefs_t* const prefs, int xxhash);
/* Default setting : 0 (no notification) */
int LZ4IO_setNotificationLevel(int level);
/* Default setting : 0 (disabled) */
-int LZ4IO_setSparseFile(int enable);
+int LZ4IO_setSparseFile(LZ4IO_prefs_t* const prefs, int enable);
/* Default setting : 0 == no content size present in frame header */
-int LZ4IO_setContentSize(int enable);
+int LZ4IO_setContentSize(LZ4IO_prefs_t* const prefs, int enable);
/* Default setting : 0 == src file preserved */
-void LZ4IO_setRemoveSrcFile(unsigned flag);
+void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag);
/* Default setting : 0 == favor compression ratio
* Note : 1 only works for high compression levels (10+) */
-void LZ4IO_favorDecSpeed(int favor);
+void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor);
+
+
+/* implement --list
+ * @return 0 on success, 1 on error */
+int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx);
#endif /* LZ4IO_H_237902873 */
diff --git a/programs/platform.h b/programs/platform.h
index c0b3840..ab8300d 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -77,16 +77,17 @@ extern "C" {
* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows
* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX
* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION
-***************************************************************/
+************************************************************** */
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
|| defined(__midipix__) || defined(__VMS))
# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \
- || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__MidnightBSD__) /* BSD distros */
+ || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__MidnightBSD__) /* BSD distros */ \
+ || defined(__HAIKU__)
# define PLATFORM_POSIX_VERSION 200112L
# else
# if defined(__linux__) || defined(__linux)
# ifndef _POSIX_C_SOURCE
-# define _POSIX_C_SOURCE 200112L /* use feature test macro */
+# define _POSIX_C_SOURCE 200809L /* use feature test macro */
# endif
# endif
# include <unistd.h> /* declares _POSIX_VERSION */
@@ -104,7 +105,7 @@ extern "C" {
/*-*********************************************
* Detect if isatty() and fileno() are available
-************************************************/
+*********************************************** */
#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__)
# include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
@@ -127,7 +128,7 @@ static __inline int IS_CONSOLE(FILE* stdStream)
/******************************
* OS-specific Includes
-******************************/
+***************************** */
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _fileno, _get_osfhandle */
diff --git a/programs/util.h b/programs/util.h
index d74db0d..733c1ca 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -33,17 +33,24 @@ extern "C" {
#include <stddef.h> /* size_t, ptrdiff_t */
#include <stdlib.h> /* malloc */
#include <string.h> /* strlen, strncpy */
-#include <stdio.h> /* fprintf */
+#include <stdio.h> /* fprintf, fileno */
+#include <assert.h>
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */
-#if defined(_MSC_VER)
+#if defined(_WIN32)
# include <sys/utime.h> /* utime */
# include <io.h> /* _chmod */
#else
# include <unistd.h> /* chown, stat */
+# if PLATFORM_POSIX_VERSION < 200809L
# include <utime.h> /* utime */
+# else
+# include <fcntl.h> /* AT_FDCWD */
+# include <sys/stat.h> /* for utimensat */
+# endif
#endif
#include <time.h> /* time */
+#include <limits.h> /* INT_MAX */
#include <errno.h>
@@ -115,6 +122,36 @@ extern "C" {
#endif
+/*-****************************************
+* stat() functions
+******************************************/
+#if defined(_MSC_VER)
+# define UTIL_TYPE_stat __stat64
+# define UTIL_stat _stat64
+# define UTIL_fstat _fstat64
+# define UTIL_STAT_MODE_ISREG(st_mode) ((st_mode) & S_IFREG)
+#elif defined(__MINGW32__) && defined (__MSVCRT__)
+# define UTIL_TYPE_stat _stati64
+# define UTIL_stat _stati64
+# define UTIL_fstat _fstati64
+# define UTIL_STAT_MODE_ISREG(st_mode) ((st_mode) & S_IFREG)
+#else
+# define UTIL_TYPE_stat stat
+# define UTIL_stat stat
+# define UTIL_fstat fstat
+# define UTIL_STAT_MODE_ISREG(st_mode) (S_ISREG(st_mode))
+#endif
+
+
+/*-****************************************
+* fileno() function
+******************************************/
+#if defined(_MSC_VER)
+# define UTIL_fileno _fileno
+#else
+# define UTIL_fileno fileno
+#endif
+
/* *************************************
* Constants
***************************************/
@@ -285,14 +322,23 @@ UTIL_STATIC int UTIL_isRegFile(const char* infilename);
UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
{
int res = 0;
- struct utimbuf timebuf;
if (!UTIL_isRegFile(filename))
return -1;
- timebuf.actime = time(NULL);
- timebuf.modtime = statbuf->st_mtime;
- res += utime(filename, &timebuf); /* set access and modification times */
+ {
+#if defined(_WIN32) || (PLATFORM_POSIX_VERSION < 200809L)
+ struct utimbuf timebuf;
+ timebuf.actime = time(NULL);
+ timebuf.modtime = statbuf->st_mtime;
+ res += utime(filename, &timebuf); /* set access and modification times */
+#else
+ struct timespec timebuf[2] = {};
+ timebuf[0].tv_nsec = UTIME_NOW;
+ timebuf[1].tv_sec = statbuf->st_mtime;
+ res += utimensat(AT_FDCWD, filename, timebuf, 0); /* set access and modification times */
+#endif
+ }
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
@@ -341,22 +387,30 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
}
+UTIL_STATIC U64 UTIL_getOpenFileSize(FILE* file)
+{
+ int r;
+ int fd;
+ struct UTIL_TYPE_stat statbuf;
+
+ fd = UTIL_fileno(file);
+ if (fd < 0) {
+ perror("fileno");
+ exit(1);
+ }
+ r = UTIL_fstat(fd, &statbuf);
+ if (r || !UTIL_STAT_MODE_ISREG(statbuf.st_mode)) return 0; /* No good... */
+ return (U64)statbuf.st_size;
+}
+
+
UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
{
int r;
-#if defined(_MSC_VER)
- struct __stat64 statbuf;
- r = _stat64(infilename, &statbuf);
- if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
-#elif defined(__MINGW32__) && defined (__MSVCRT__)
- struct _stati64 statbuf;
- r = _stati64(infilename, &statbuf);
- if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
-#else
- struct stat statbuf;
- r = stat(infilename, &statbuf);
- if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
-#endif
+ struct UTIL_TYPE_stat statbuf;
+
+ r = UTIL_stat(infilename, &statbuf);
+ if (r || !UTIL_STAT_MODE_ISREG(statbuf.st_mode)) return 0; /* No good... */
return (U64)statbuf.st_size;
}
@@ -375,9 +429,9 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi
* A modified version of realloc().
* If UTIL_realloc() fails the original block is freed.
*/
-UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
+UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size)
{
- void *newptr = realloc(ptr, size);
+ void* const newptr = realloc(ptr, size);
if (newptr) return newptr;
free(ptr);
return NULL;
@@ -387,14 +441,14 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
#ifdef _WIN32
# define UTIL_HAS_CREATEFILELIST
-UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
+UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd)
{
char* path;
- int dirLength, fnameLength, pathLength, nbFiles = 0;
+ size_t dirLength, nbFiles = 0;
WIN32_FIND_DATAA cFile;
HANDLE hFile;
- dirLength = (int)strlen(dirName);
+ dirLength = strlen(dirName);
path = (char*) malloc(dirLength + 3);
if (!path) return 0;
@@ -411,7 +465,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
free(path);
do {
- fnameLength = (int)strlen(cFile.cFileName);
+ size_t pathLength;
+ int const fnameLength = (int)strlen(cFile.cFileName);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { FindClose(hFile); return 0; }
memcpy(path, dirName, dirLength);
@@ -443,7 +498,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
} while (FindNextFileA(hFile, &cFile));
FindClose(hFile);
- return nbFiles;
+ assert(nbFiles < INT_MAX);
+ return (int)nbFiles;
}
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
@@ -451,12 +507,11 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
# include <dirent.h> /* opendir, readdir */
# include <string.h> /* strerror, memcpy */
-UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
+UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd)
{
- DIR *dir;
- struct dirent *entry;
- char* path;
- int dirLength, fnameLength, pathLength, nbFiles = 0;
+ DIR* dir;
+ struct dirent * entry;
+ int dirLength, nbFiles = 0;
if (!(dir = opendir(dirName))) {
fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
@@ -466,6 +521,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
dirLength = (int)strlen(dirName);
errno = 0;
while ((entry = readdir(dir)) != NULL) {
+ char* path;
+ int fnameLength, pathLength;
if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue;
fnameLength = (int)strlen(entry->d_name);
@@ -508,7 +565,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
#else
-UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
+UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd)
{
(void)bufStart; (void)bufEnd; (void)pos;
fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
@@ -523,38 +580,41 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
*/
-UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb)
+UTIL_STATIC const char**
+UTIL_createFileList(const char** inputNames, unsigned inputNamesNb,
+ char** allocatedBuffer, unsigned* allocatedNamesNb)
{
size_t pos;
unsigned i, nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
- char* bufend = buf + LIST_SIZE_INCREASE;
+ size_t bufSize = LIST_SIZE_INCREASE;
const char** fileTable;
if (!buf) return NULL;
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
if (!UTIL_isDirectory(inputNames[i])) {
- size_t const len = strlen(inputNames[i]);
- if (buf + pos + len >= bufend) {
- ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
- buf = (char*)UTIL_realloc(buf, newListSize);
- bufend = buf + newListSize;
+ size_t const len = strlen(inputNames[i]) + 1; /* include nul char */
+ if (pos + len >= bufSize) {
+ while (pos + len >= bufSize) bufSize += LIST_SIZE_INCREASE;
+ buf = (char*)UTIL_realloc(buf, bufSize);
if (!buf) return NULL;
}
- if (buf + pos + len < bufend) {
- strncpy(buf + pos, inputNames[i], bufend - (buf + pos));
- pos += len + 1;
- nbFiles++;
- }
+ assert(pos + len < bufSize);
+ memcpy(buf + pos, inputNames[i], len);
+ pos += len;
+ nbFiles++;
} else {
- nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend);
+ char* bufend = buf + bufSize;
+ nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend);
if (buf == NULL) return NULL;
+ assert(bufend > buf);
+ bufSize = (size_t)(bufend - buf);
} }
if (nbFiles == 0) { free(buf); return NULL; }
- fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
+ fileTable = (const char**)malloc(((size_t)nbFiles+1) * sizeof(const char*));
if (!fileTable) { free(buf); return NULL; }
for (i=0, pos=0; i<nbFiles; i++) {
@@ -562,7 +622,11 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
pos += strlen(fileTable[i]) + 1;
}
- if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
+ if (pos > bufSize) {
+ free(buf);
+ free((void*)fileTable);
+ return NULL;
+ } /* can this happen ? */
*allocatedBuffer = buf;
*allocatedNamesNb = nbFiles;
@@ -571,7 +635,8 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
}
-UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
+UTIL_STATIC void
+UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
{
if (allocatedBuffer) free(allocatedBuffer);
if (filenameTable) free((void*)filenameTable);
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-03 08:05:21 +0000