diff options
| author | Haibo Huang <hhb@google.com> | 2019-06-11 14:59:02 -0700 |
|---|---|---|
| committer | Haibo Huang <hhb@google.com> | 2019-06-11 23:49:27 +0000 |
| commit | 880009beeff9e3c1723d8806caa063dc1cee9a0c (patch) | |
| tree | bb61070a84573cd064524e9c09f02f748c070852 /dist/shell.c | |
| parent | 6d1591ee346f7582e92fe4ba3509433da636fe58 (diff) | |
sqlite: Upgrade to SQLite 3.28.0
Downloaded from
https://sqlite.org/2019/sqlite-autoconf-3280000.tar.gz
$ sha1sum sqlite-autoconf-3280000.tar.gz
01b9d8fc77085e144dddc87456c9783e53d09a53 sqlite-autoconf-3280000.tar.gz
dist/orig contains the stock sqlite3 code, as packaged in the tar.gz file above.
dist contains a copy of dist/orig, but with the Android.patch file applied.
The Android specific patch didn't apply cleanly due to merge conflict in shell.c. The conflict was resolved and the patch was regenerated.
More details about the release are available here:
https://www.sqlite.org/releaselog/3_28_0.html
Test: code compiles and device boots with no problems.
Test: select sqlite_version() - returns 3.28.0
Test: atest cts/SQLiteDatabaseTest all passed.
Test: atest cts/SQLiteSecurityTest
Bug: 135048770
Change-Id: Idf38a3adace9f4bd4feab4012c9cc4de3a1c501c
Diffstat (limited to 'dist/shell.c')
| -rw-r--r-- | dist/shell.c | 4424 |
1 files changed, 3137 insertions, 1287 deletions
diff --git a/dist/shell.c b/dist/shell.c index 73f8b85..b06efb5 100644 --- a/dist/shell.c +++ b/dist/shell.c @@ -103,12 +103,15 @@ typedef unsigned char u8; #if (!defined(_WIN32) && !defined(WIN32)) || defined(__MINGW32__) # include <unistd.h> # include <dirent.h> +# define GETPID getpid # if defined(__MINGW32__) # define DIRENT dirent # ifndef S_ISLNK # define S_ISLNK(mode) (0) # endif # endif +#else +# define GETPID (int)GetCurrentProcessId #endif #include <sys/types.h> #include <sys/stat.h> @@ -156,6 +159,12 @@ typedef unsigned char u8; # ifndef access # define access(f,m) _access((f),(m)) # endif +# ifndef unlink +# define unlink _unlink +# endif +# ifndef strdup +# define strdup _strdup +# endif # undef popen # define popen _popen # undef pclose @@ -457,6 +466,12 @@ void utf8_printf(FILE *out, const char *zFormat, ...){ # define raw_printf fprintf #endif +/* Indicate out-of-memory and exit. */ +static void shell_out_of_memory(void){ + raw_printf(stderr,"Error: out of memory\n"); + exit(1); +} + /* ** Write I/O traces to the following stream. */ @@ -580,7 +595,7 @@ static char *local_getline(char *zLine, FILE *in){ if( n+100>nLine ){ nLine = nLine*2 + 100; zLine = realloc(zLine, nLine); - if( zLine==0 ) return 0; + if( zLine==0 ) shell_out_of_memory(); } if( fgets(&zLine[n], nLine - n, in)==0 ){ if( n==0 ){ @@ -607,10 +622,7 @@ static char *local_getline(char *zLine, FILE *in){ int nTrans = strlen30(zTrans)+1; if( nTrans>nLine ){ zLine = realloc(zLine, nTrans); - if( zLine==0 ){ - sqlite3_free(zTrans); - return 0; - } + if( zLine==0 ) shell_out_of_memory(); } memcpy(zLine, zTrans, nTrans); sqlite3_free(zTrans); @@ -757,10 +769,7 @@ static void appendText(ShellText *p, char const *zAppend, char quote){ if( p->n+len>=p->nAlloc ){ p->nAlloc = p->nAlloc*2 + len + 20; p->z = realloc(p->z, p->nAlloc); - if( p->z==0 ){ - memset(p, 0, sizeof(*p)); - return; - } + if( p->z==0 ) shell_out_of_memory(); } if( quote ){ @@ -789,45 +798,12 @@ static void appendText(ShellText *p, char const *zAppend, char quote){ ** Return '"' if quoting is required. Return 0 if no quoting is required. */ static char quoteChar(const char *zName){ - /* All SQLite keywords, in alphabetical order */ - static const char *azKeywords[] = { - "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS", - "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", - "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT", - "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", - "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE", - "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH", - "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN", - "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF", - "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", - "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", - "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL", - "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA", - "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP", - "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT", - "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP", - "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE", - "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE", - "WITH", "WITHOUT", - }; - int i, lwr, upr, mid, c; + int i; if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; for(i=0; zName[i]; i++){ if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; } - lwr = 0; - upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1; - while( lwr<=upr ){ - mid = (lwr+upr)/2; - c = sqlite3_stricmp(azKeywords[mid], zName); - if( c==0 ) return '"'; - if( c<0 ){ - lwr = mid+1; - }else{ - upr = mid-1; - } - } - return 0; + return sqlite3_keyword_check(zName, i) ? '"' : 0; } /* @@ -1003,7 +979,10 @@ static void shellAddSchemaName( ** We need several data types from the Windows SDK header. */ +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif + #include "windows.h" /* @@ -1347,7 +1326,7 @@ INT closedir( ** ****************************************************************************** ** -** This SQLite extension implements a functions that compute SHA1 hashes. +** This SQLite extension implements functions that compute SHA3 hashes. ** Two SQL functions are implemented: ** ** sha3(X,SIZE) @@ -2145,6 +2124,9 @@ SQLITE_EXTENSION_INIT1 # include <direct.h> /* # include "test_windirent.h" */ # define dirent DIRENT +# ifndef chmod +# define chmod _chmod +# endif # ifndef stat # define stat _stat # endif @@ -2155,26 +2137,62 @@ SQLITE_EXTENSION_INIT1 #include <errno.h> +/* +** Structure of the fsdir() table-valued function +*/ + /* 0 1 2 3 4 5 */ #define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" +#define FSDIR_COLUMN_NAME 0 /* Name of the file */ +#define FSDIR_COLUMN_MODE 1 /* Access mode */ +#define FSDIR_COLUMN_MTIME 2 /* Last modification time */ +#define FSDIR_COLUMN_DATA 3 /* File content */ +#define FSDIR_COLUMN_PATH 4 /* Path to top of search */ +#define FSDIR_COLUMN_DIR 5 /* Path is relative to this directory */ + /* ** Set the result stored by context ctx to a blob containing the -** contents of file zName. +** contents of file zName. Or, leave the result unchanged (NULL) +** if the file does not exist or is unreadable. +** +** If the file exceeds the SQLite blob size limit, through an +** SQLITE_TOOBIG error. +** +** Throw an SQLITE_IOERR if there are difficulties pulling the file +** off of disk. */ static void readFileContents(sqlite3_context *ctx, const char *zName){ FILE *in; - long nIn; + sqlite3_int64 nIn; void *pBuf; + sqlite3 *db; + int mxBlob; in = fopen(zName, "rb"); - if( in==0 ) return; + if( in==0 ){ + /* File does not exist or is unreadable. Leave the result set to NULL. */ + return; + } fseek(in, 0, SEEK_END); nIn = ftell(in); rewind(in); - pBuf = sqlite3_malloc( nIn ); - if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ - sqlite3_result_blob(ctx, pBuf, nIn, sqlite3_free); + db = sqlite3_context_db_handle(ctx); + mxBlob = sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1); + if( nIn>mxBlob ){ + sqlite3_result_error_code(ctx, SQLITE_TOOBIG); + fclose(in); + return; + } + pBuf = sqlite3_malloc64( nIn ? nIn : 1 ); + if( pBuf==0 ){ + sqlite3_result_error_nomem(ctx); + fclose(in); + return; + } + if( nIn==(sqlite3_int64)fread(pBuf, 1, (size_t)nIn, in) ){ + sqlite3_result_blob64(ctx, pBuf, nIn, sqlite3_free); }else{ + sqlite3_result_error_code(ctx, SQLITE_IOERR); sqlite3_free(pBuf); } fclose(in); @@ -2211,19 +2229,110 @@ static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ va_end(ap); } +#if defined(_WIN32) +/* +** This function is designed to convert a Win32 FILETIME structure into the +** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC). +*/ +static sqlite3_uint64 fileTimeToUnixTime( + LPFILETIME pFileTime +){ + SYSTEMTIME epochSystemTime; + ULARGE_INTEGER epochIntervals; + FILETIME epochFileTime; + ULARGE_INTEGER fileIntervals; + + memset(&epochSystemTime, 0, sizeof(SYSTEMTIME)); + epochSystemTime.wYear = 1970; + epochSystemTime.wMonth = 1; + epochSystemTime.wDay = 1; + SystemTimeToFileTime(&epochSystemTime, &epochFileTime); + epochIntervals.LowPart = epochFileTime.dwLowDateTime; + epochIntervals.HighPart = epochFileTime.dwHighDateTime; + + fileIntervals.LowPart = pFileTime->dwLowDateTime; + fileIntervals.HighPart = pFileTime->dwHighDateTime; + + return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; +} + +/* +** This function attempts to normalize the time values found in the stat() +** buffer to UTC. This is necessary on Win32, where the runtime library +** appears to return these values as local times. +*/ +static void statTimesToUtc( + const char *zPath, + struct stat *pStatBuf +){ + HANDLE hFindFile; + WIN32_FIND_DATAW fd; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); + if( zUnicodeName ){ + memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); + hFindFile = FindFirstFileW(zUnicodeName, &fd); + if( hFindFile!=NULL ){ + pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); + pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); + pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); + FindClose(hFindFile); + } + sqlite3_free(zUnicodeName); + } +} +#endif + +/* +** This function is used in place of stat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls stat(). +*/ +static int fileStat( + const char *zPath, + struct stat *pStatBuf +){ +#if defined(_WIN32) + int rc = stat(zPath, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + return rc; +#else + return stat(zPath, pStatBuf); +#endif +} + +/* +** This function is used in place of lstat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls lstat(). +*/ +static int fileLinkStat( + const char *zPath, + struct stat *pStatBuf +){ +#if defined(_WIN32) + int rc = lstat(zPath, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + return rc; +#else + return lstat(zPath, pStatBuf); +#endif +} + /* ** Argument zFile is the name of a file that will be created and/or written ** by SQL function writefile(). This function ensures that the directory ** zFile will be written to exists, creating it if required. The permissions -** for any path components created by this function are set to (mode&0777). +** for any path components created by this function are set in accordance +** with the current umask. ** ** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise, ** SQLITE_OK is returned if the directory is successfully created, or ** SQLITE_ERROR otherwise. */ static int makeDirectory( - const char *zFile, - mode_t mode + const char *zFile ){ char *zCopy = sqlite3_mprintf("%s", zFile); int rc = SQLITE_OK; @@ -2242,9 +2351,9 @@ static int makeDirectory( if( i==nCopy ) break; zCopy[i] = '\0'; - rc2 = stat(zCopy, &sStat); + rc2 = fileStat(zCopy, &sStat); if( rc2!=0 ){ - if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR; + if( mkdir(zCopy, 0777) ) rc = SQLITE_ERROR; }else{ if( !S_ISDIR(sStat.st_mode) ) rc = SQLITE_ERROR; } @@ -2284,7 +2393,7 @@ static int writeFile( ** to do so using chmod(), it is not an error. */ struct stat sStat; if( errno!=EEXIST - || 0!=stat(zFile, &sStat) + || 0!=fileStat(zFile, &sStat) || !S_ISDIR(sStat.st_mode) || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile, mode&0777)) ){ @@ -2322,15 +2431,23 @@ static int writeFile( SYSTEMTIME currentTime; LONGLONG intervals; HANDLE hFile; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + GetSystemTime(¤tTime); SystemTimeToFileTime(¤tTime, &lastAccess); intervals = Int32x32To64(mtime, 10000000) + 116444736000000000; lastWrite.dwLowDateTime = (DWORD)intervals; lastWrite.dwHighDateTime = intervals >> 32; - hFile = CreateFile( - zFile, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING, + zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile); + if( zUnicodeName==0 ){ + return 1; + } + hFile = CreateFileW( + zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL ); + sqlite3_free(zUnicodeName); if( hFile!=INVALID_HANDLE_VALUE ){ BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite); CloseHandle(hFile); @@ -2338,7 +2455,7 @@ static int writeFile( }else{ return 1; } -#elif defined(AT_FDCWD) && 0 /* utimensat() is not univerally available */ +#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ /* Recent unix */ struct timespec times[2]; times[0].tv_nsec = times[1].tv_nsec = 0; @@ -2394,7 +2511,7 @@ static void writefileFunc( res = writeFile(context, zFile, argv[1], mode, mtime); if( res==1 && errno==ENOENT ){ - if( makeDirectory(zFile, mode)==SQLITE_OK ){ + if( makeDirectory(zFile)==SQLITE_OK ){ res = writeFile(context, zFile, argv[1], mode, mtime); } } @@ -2538,10 +2655,12 @@ static void fsdirResetCursor(fsdir_cursor *pCur){ sqlite3_free(pLvl->zDir); } sqlite3_free(pCur->zPath); + sqlite3_free(pCur->aLvl); pCur->aLvl = 0; pCur->zPath = 0; pCur->zBase = 0; pCur->nBase = 0; + pCur->nLvl = 0; pCur->iLvl = -1; pCur->iRowid = 1; } @@ -2553,7 +2672,6 @@ static int fsdirClose(sqlite3_vtab_cursor *cur){ fsdir_cursor *pCur = (fsdir_cursor*)cur; fsdirResetCursor(pCur); - sqlite3_free(pCur->aLvl); sqlite3_free(pCur); return SQLITE_OK; } @@ -2584,8 +2702,8 @@ static int fsdirNext(sqlite3_vtab_cursor *cur){ FsdirLevel *pLvl; if( iNew>=pCur->nLvl ){ int nNew = iNew+1; - int nByte = nNew*sizeof(FsdirLevel); - FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc(pCur->aLvl, nByte); + sqlite3_int64 nByte = nNew*sizeof(FsdirLevel); + FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc64(pCur->aLvl, nByte); if( aNew==0 ) return SQLITE_NOMEM; memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl)); pCur->aLvl = aNew; @@ -2614,7 +2732,7 @@ static int fsdirNext(sqlite3_vtab_cursor *cur){ sqlite3_free(pCur->zPath); pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name); if( pCur->zPath==0 ) return SQLITE_NOMEM; - if( lstat(pCur->zPath, &pCur->sStat) ){ + if( fileLinkStat(pCur->zPath, &pCur->sStat) ){ fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); return SQLITE_ERROR; } @@ -2644,20 +2762,20 @@ static int fsdirColumn( ){ fsdir_cursor *pCur = (fsdir_cursor*)cur; switch( i ){ - case 0: { /* name */ + case FSDIR_COLUMN_NAME: { sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT); break; } - case 1: /* mode */ + case FSDIR_COLUMN_MODE: sqlite3_result_int64(ctx, pCur->sStat.st_mode); break; - case 2: /* mtime */ + case FSDIR_COLUMN_MTIME: sqlite3_result_int64(ctx, pCur->sStat.st_mtime); break; - case 3: { /* data */ + case FSDIR_COLUMN_DATA: { mode_t m = pCur->sStat.st_mode; if( S_ISDIR(m) ){ sqlite3_result_null(ctx); @@ -2665,7 +2783,7 @@ static int fsdirColumn( }else if( S_ISLNK(m) ){ char aStatic[64]; char *aBuf = aStatic; - int nBuf = 64; + sqlite3_int64 nBuf = 64; int n; while( 1 ){ @@ -2673,7 +2791,7 @@ static int fsdirColumn( if( n<nBuf ) break; if( aBuf!=aStatic ) sqlite3_free(aBuf); nBuf = nBuf*2; - aBuf = sqlite3_malloc(nBuf); + aBuf = sqlite3_malloc64(nBuf); if( aBuf==0 ){ sqlite3_result_error_nomem(ctx); return SQLITE_NOMEM; @@ -2687,6 +2805,12 @@ static int fsdirColumn( readFileContents(ctx, pCur->zPath); } } + case FSDIR_COLUMN_PATH: + default: { + /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters. + ** always return their values as NULL */ + break; + } } return SQLITE_OK; } @@ -2713,6 +2837,9 @@ static int fsdirEof(sqlite3_vtab_cursor *cur){ /* ** xFilter callback. +** +** idxNum==1 PATH parameter only +** idxNum==2 Both PATH and DIR supplied */ static int fsdirFilter( sqlite3_vtab_cursor *cur, @@ -2748,7 +2875,7 @@ static int fsdirFilter( if( pCur->zPath==0 ){ return SQLITE_NOMEM; } - if( lstat(pCur->zPath, &pCur->sStat) ){ + if( fileLinkStat(pCur->zPath, &pCur->sStat) ){ fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); return SQLITE_ERROR; } @@ -2765,40 +2892,63 @@ static int fsdirFilter( ** In this implementation idxNum is used to represent the ** query plan. idxStr is unused. ** -** The query plan is represented by bits in idxNum: +** The query plan is represented by values of idxNum: ** -** (1) start = $value -- constraint exists -** (2) stop = $value -- constraint exists -** (4) step = $value -- constraint exists -** (8) output in descending order +** (1) The path value is supplied by argv[0] +** (2) Path is in argv[0] and dir is in argv[1] */ static int fsdirBestIndex( sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo ){ int i; /* Loop over constraints */ - int idx4 = -1; - int idx5 = -1; + int idxPath = -1; /* Index in pIdxInfo->aConstraint of PATH= */ + int idxDir = -1; /* Index in pIdxInfo->aConstraint of DIR= */ + int seenPath = 0; /* True if an unusable PATH= constraint is seen */ + int seenDir = 0; /* True if an unusable DIR= constraint is seen */ const struct sqlite3_index_constraint *pConstraint; (void)tab; pConstraint = pIdxInfo->aConstraint; for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - if( pConstraint->iColumn==4 ) idx4 = i; - if( pConstraint->iColumn==5 ) idx5 = i; + switch( pConstraint->iColumn ){ + case FSDIR_COLUMN_PATH: { + if( pConstraint->usable ){ + idxPath = i; + seenPath = 0; + }else if( idxPath<0 ){ + seenPath = 1; + } + break; + } + case FSDIR_COLUMN_DIR: { + if( pConstraint->usable ){ + idxDir = i; + seenDir = 0; + }else if( idxDir<0 ){ + seenDir = 1; + } + break; + } + } + } + if( seenPath || seenDir ){ + /* If input parameters are unusable, disallow this plan */ + return SQLITE_CONSTRAINT; } - if( idx4<0 ){ + if( idxPath<0 ){ pIdxInfo->idxNum = 0; - pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50); + /* The pIdxInfo->estimatedCost should have been initialized to a huge + ** number. Leave it unchanged. */ + pIdxInfo->estimatedRows = 0x7fffffff; }else{ - pIdxInfo->aConstraintUsage[idx4].omit = 1; - pIdxInfo->aConstraintUsage[idx4].argvIndex = 1; - if( idx5>=0 ){ - pIdxInfo->aConstraintUsage[idx5].omit = 1; - pIdxInfo->aConstraintUsage[idx5].argvIndex = 2; + pIdxInfo->aConstraintUsage[idxPath].omit = 1; + pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1; + if( idxDir>=0 ){ + pIdxInfo->aConstraintUsage[idxDir].omit = 1; + pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2; pIdxInfo->idxNum = 2; pIdxInfo->estimatedCost = 10.0; }else{ @@ -2837,7 +2987,8 @@ static int fsdirRegister(sqlite3 *db){ 0, /* xRename */ 0, /* xSavepoint */ 0, /* xRelease */ - 0 /* xRollbackTo */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ }; int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0); @@ -2939,6 +3090,7 @@ struct completion_cursor { char *zPrefix; /* The prefix for the word we want to complete */ char *zLine; /* The whole that we want to complete */ const char *zCurrentRow; /* Current output row */ + int szRow; /* Length of the zCurrentRow string */ sqlite3_stmt *pStmt; /* Current statement */ sqlite3_int64 iRowid; /* The rowid */ int ePhase; /* Current phase */ @@ -2955,7 +3107,7 @@ struct completion_cursor { #define COMPLETION_INDEXES 5 #define COMPLETION_TRIGGERS 6 #define COMPLETION_DATABASES 7 -#define COMPLETION_TABLES 8 +#define COMPLETION_TABLES 8 /* Also VIEWs and TRIGGERs */ #define COMPLETION_COLUMNS 9 #define COMPLETION_MODULES 10 #define COMPLETION_EOF 11 @@ -3052,32 +3204,6 @@ static int completionClose(sqlite3_vtab_cursor *cur){ } /* -** All SQL keywords understood by SQLite -*/ -static const char *completionKwrds[] = { - "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS", - "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", - "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT", - "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", - "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE", - "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH", - "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN", - "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF", - "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", - "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", - "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL", - "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA", - "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP", - "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT", - "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP", - "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE", - "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE", - "WITH", "WITHOUT", -}; -#define completionKwCount \ - (int)(sizeof(completionKwrds)/sizeof(completionKwrds[0])) - -/* ** Advance a completion_cursor to its next row of output. ** ** The ->ePhase, ->j, and ->pStmt fields of the completion_cursor object @@ -3099,11 +3225,11 @@ static int completionNext(sqlite3_vtab_cursor *cur){ while( pCur->ePhase!=COMPLETION_EOF ){ switch( pCur->ePhase ){ case COMPLETION_KEYWORDS: { - if( pCur->j >= completionKwCount ){ + if( pCur->j >= sqlite3_keyword_count() ){ pCur->zCurrentRow = 0; pCur->ePhase = COMPLETION_DATABASES; }else{ - pCur->zCurrentRow = completionKwrds[pCur->j++]; + sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow); } iCol = -1; break; @@ -3127,8 +3253,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); zSql = sqlite3_mprintf( "%z%s" - "SELECT name FROM \"%w\".sqlite_master" - " WHERE type='table'", + "SELECT name FROM \"%w\".sqlite_master", zSql, zSep, zDb ); if( zSql==0 ) return SQLITE_NOMEM; @@ -3176,6 +3301,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){ if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){ /* Extract the next row of content */ pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol); + pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol); }else{ /* When all rows are finished, advance to the next phase */ sqlite3_finalize(pCur->pStmt); @@ -3185,7 +3311,9 @@ static int completionNext(sqlite3_vtab_cursor *cur){ } } if( pCur->nPrefix==0 ) break; - if( sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 ){ + if( pCur->nPrefix<=pCur->szRow + && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 + ){ break; } } @@ -3205,7 +3333,7 @@ static int completionColumn( completion_cursor *pCur = (completion_cursor*)cur; switch( i ){ case COMPLETION_COLUMN_CANDIDATE: { - sqlite3_result_text(ctx, pCur->zCurrentRow, -1, SQLITE_TRANSIENT); + sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT); break; } case COMPLETION_COLUMN_PREFIX: { @@ -3265,7 +3393,7 @@ static int completionFilter( pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); if( pCur->zPrefix==0 ) return SQLITE_NOMEM; } - iArg++; + iArg = 1; } if( idxNum & 2 ){ pCur->nLine = sqlite3_value_bytes(argv[iArg]); @@ -3273,7 +3401,6 @@ static int completionFilter( pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); if( pCur->zLine==0 ) return SQLITE_NOMEM; } - iArg++; } if( pCur->zLine!=0 && pCur->zPrefix==0 ){ int i = pCur->nLine; @@ -3369,7 +3496,8 @@ static sqlite3_module completionModule = { 0, /* xRename */ 0, /* xSavepoint */ 0, /* xRelease */ - 0 /* xRollbackTo */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -3967,6 +4095,117 @@ int sqlite3_appendvfs_init( } /************************* End ../ext/misc/appendvfs.c ********************/ +/************************* Begin ../ext/misc/memtrace.c ******************/ +/* +** 2019-01-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_MALLOC +** mechanism to add a tracing layer on top of SQLite. If this extension +** is registered prior to sqlite3_initialize(), it will cause all memory +** allocation activities to be logged on standard output, or to some other +** FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --memtrace option of the +** command-line shell. +*/ +#include <assert.h> +#include <string.h> +#include <stdio.h> + +/* The original memory allocation routines */ +static sqlite3_mem_methods memtraceBase; +static FILE *memtraceOut; + +/* Methods that trace memory allocations */ +static void *memtraceMalloc(int n){ + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: allocate %d bytes\n", + memtraceBase.xRoundup(n)); + } + return memtraceBase.xMalloc(n); +} +static void memtraceFree(void *p){ + if( p==0 ) return; + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: free %d bytes\n", memtraceBase.xSize(p)); + } + memtraceBase.xFree(p); +} +static void *memtraceRealloc(void *p, int n){ + if( p==0 ) return memtraceMalloc(n); + if( n==0 ){ + memtraceFree(p); + return 0; + } + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: resize %d -> %d bytes\n", + memtraceBase.xSize(p), memtraceBase.xRoundup(n)); + } + return memtraceBase.xRealloc(p, n); +} +static int memtraceSize(void *p){ + return memtraceBase.xSize(p); +} +static int memtraceRoundup(int n){ + return memtraceBase.xRoundup(n); +} +static int memtraceInit(void *p){ + return memtraceBase.xInit(p); +} +static void memtraceShutdown(void *p){ + memtraceBase.xShutdown(p); +} + +/* The substitute memory allocator */ +static sqlite3_mem_methods ersaztMethods = { + memtraceMalloc, + memtraceFree, + memtraceRealloc, + memtraceSize, + memtraceRoundup, + memtraceInit, + memtraceShutdown, + 0 +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3MemTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &ersaztMethods); + } + } + memtraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3MemTraceDeactivate(void){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + memset(&memtraceBase, 0, sizeof(memtraceBase)); + } + } + memtraceOut = 0; + return rc; +} + +/************************* End ../ext/misc/memtrace.c ********************/ #ifdef SQLITE_HAVE_ZLIB /************************* Begin ../ext/misc/zipfile.c ******************/ /* @@ -4000,29 +4239,48 @@ SQLITE_EXTENSION_INIT1 #include <string.h> #include <assert.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#if !defined(_WIN32) && !defined(WIN32) -# include <unistd.h> -# include <dirent.h> -# include <utime.h> -#else -# include <io.h> -#endif -#include <time.h> -#include <errno.h> - #include <zlib.h> #ifndef SQLITE_OMIT_VIRTUALTABLE #ifndef SQLITE_AMALGAMATION + /* typedef sqlite3_int64 i64; */ /* typedef unsigned char u8; */ typedef unsigned short u16; typedef unsigned long u32; #define MIN(a,b) ((a)<(b) ? (a) : (b)) + +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif + +#endif /* SQLITE_AMALGAMATION */ + +/* +** Definitions for mode bitmasks S_IFDIR, S_IFREG and S_IFLNK. +** +** In some ways it would be better to obtain these values from system +** header files. But, the dependency is undesirable and (a) these +** have been stable for decades, (b) the values are part of POSIX and +** are also made explicit in [man stat], and (c) are part of the +** file format for zip archives. +*/ +#ifndef S_IFDIR +# define S_IFDIR 0040000 +#endif +#ifndef S_IFREG +# define S_IFREG 0100000 +#endif +#ifndef S_IFLNK +# define S_IFLNK 0120000 #endif static const char ZIPFILE_SCHEMA[] = @@ -4063,6 +4321,9 @@ static const char ZIPFILE_SCHEMA[] = ** ** ZIPFILE_SIGNATURE_LFH: ** First 4 bytes of a valid LFH record. +** +** ZIPFILE_SIGNATURE_EOCD +** First 4 bytes of a valid EOCD record. */ #define ZIPFILE_EXTRA_TIMESTAMP 0x5455 #define ZIPFILE_NEWENTRY_MADEBY ((3<<8) + 30) @@ -4071,22 +4332,14 @@ static const char ZIPFILE_SCHEMA[] = #define ZIPFILE_SIGNATURE_CDS 0x02014b50 #define ZIPFILE_SIGNATURE_LFH 0x04034b50 #define ZIPFILE_SIGNATURE_EOCD 0x06054b50 -#define ZIPFILE_LFH_FIXED_SZ 30 /* -** Set the error message contained in context ctx to the results of -** vprintf(zFmt, ...). +** The sizes of the fixed-size part of each of the three main data +** structures in a zip archive. */ -static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ - char *zMsg = 0; - va_list ap; - va_start(ap, zFmt); - zMsg = sqlite3_vmprintf(zFmt, ap); - sqlite3_result_error(ctx, zMsg, -1); - sqlite3_free(zMsg); - va_end(ap); -} - +#define ZIPFILE_LFH_FIXED_SZ 30 +#define ZIPFILE_EOCD_FIXED_SZ 22 +#define ZIPFILE_CDS_FIXED_SZ 46 /* *** 4.3.16 End of central directory record: @@ -4192,47 +4445,39 @@ struct ZipfileLFH { typedef struct ZipfileEntry ZipfileEntry; struct ZipfileEntry { - char *zPath; /* Path of zipfile entry */ - u8 *aCdsEntry; /* Buffer containing entire CDS entry */ - int nCdsEntry; /* Size of buffer aCdsEntry[] in bytes */ - int bDeleted; /* True if entry has been deleted */ + ZipfileCDS cds; /* Parsed CDS record */ + u32 mUnixTime; /* Modification time, in UNIX format */ + u8 *aExtra; /* cds.nExtra+cds.nComment bytes of extra data */ + i64 iDataOff; /* Offset to data in file (if aData==0) */ + u8 *aData; /* cds.szCompressed bytes of compressed data */ ZipfileEntry *pNext; /* Next element in in-memory CDS */ }; /* -** Cursor type for recursively iterating through a directory structure. +** Cursor type for zipfile tables. */ typedef struct ZipfileCsr ZipfileCsr; struct ZipfileCsr { sqlite3_vtab_cursor base; /* Base class - must be first */ i64 iId; /* Cursor ID */ - int bEof; /* True when at EOF */ + u8 bEof; /* True when at EOF */ + u8 bNoop; /* If next xNext() call is no-op */ /* Used outside of write transactions */ FILE *pFile; /* Zip file */ i64 iNextOff; /* Offset of next record in central directory */ ZipfileEOCD eocd; /* Parse of central directory record */ - /* Used inside write transactions */ - ZipfileEntry *pCurrent; - - ZipfileCDS cds; /* Central Directory Structure */ - ZipfileLFH lfh; /* Local File Header for current entry */ - i64 iDataOff; /* Offset in zipfile to data */ - u32 mTime; /* Extended mtime value */ - int flags; /* Flags byte (see below for bits) */ + ZipfileEntry *pFreeEntry; /* Free this list when cursor is closed or reset */ + ZipfileEntry *pCurrent; /* Current entry */ ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */ }; -/* -** Values for ZipfileCsr.flags. -*/ -#define ZIPFILE_MTIME_VALID 0x0001 - typedef struct ZipfileTab ZipfileTab; struct ZipfileTab { sqlite3_vtab base; /* Base class - must be first */ char *zFile; /* Zip file this table accesses (may be NULL) */ + sqlite3 *db; /* Host database connection */ u8 *aBuffer; /* Temporary buffer used for various tasks */ ZipfileCsr *pCsrList; /* List of cursors */ @@ -4246,17 +4491,33 @@ struct ZipfileTab { i64 szOrig; /* Size of archive at start of transaction */ }; +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ +static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ + char *zMsg = 0; + va_list ap; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + va_end(ap); +} + +/* +** If string zIn is quoted, dequote it in place. Otherwise, if the string +** is not quoted, do nothing. +*/ static void zipfileDequote(char *zIn){ char q = zIn[0]; if( q=='"' || q=='\'' || q=='`' || q=='[' ){ - char c; int iIn = 1; int iOut = 0; if( q=='[' ) q = ']'; - while( (c = zIn[iIn++]) ){ - if( c==q ){ - if( zIn[iIn++]!=q ) break; - } + while( ALWAYS(zIn[iIn]) ){ + char c = zIn[iIn++]; + if( c==q && zIn[iIn++]!=q ) break; zIn[iOut++] = c; } zIn[iOut] = '\0'; @@ -4284,6 +4545,21 @@ static int zipfileConnect( ZipfileTab *pNew = 0; int rc; + /* If the table name is not "zipfile", require that the argument be + ** specified. This stops zipfile tables from being created as: + ** + ** CREATE VIRTUAL TABLE zzz USING zipfile(); + ** + ** It does not prevent: + ** + ** CREATE VIRTUAL TABLE zipfile USING zipfile(); + */ + assert( 0==sqlite3_stricmp(argv[0], "zipfile") ); + if( (0!=sqlite3_stricmp(argv[2], "zipfile") && argc<4) || argc>4 ){ + *pzErr = sqlite3_mprintf("zipfile constructor requires one argument"); + return SQLITE_ERROR; + } + if( argc>3 ){ zFile = argv[3]; nFile = (int)strlen(zFile)+1; @@ -4291,9 +4567,10 @@ static int zipfileConnect( rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA); if( rc==SQLITE_OK ){ - pNew = (ZipfileTab*)sqlite3_malloc(nByte+nFile); + pNew = (ZipfileTab*)sqlite3_malloc64((sqlite3_int64)nByte+nFile); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, nByte+nFile); + pNew->db = db; pNew->aBuffer = (u8*)&pNew[1]; if( zFile ){ pNew->zFile = (char*)&pNew->aBuffer[ZIPFILE_BUFFER_SIZE]; @@ -4306,9 +4583,42 @@ static int zipfileConnect( } /* +** Free the ZipfileEntry structure indicated by the only argument. +*/ +static void zipfileEntryFree(ZipfileEntry *p){ + if( p ){ + sqlite3_free(p->cds.zFile); + sqlite3_free(p); + } +} + +/* +** Release resources that should be freed at the end of a write +** transaction. +*/ +static void zipfileCleanupTransaction(ZipfileTab *pTab){ + ZipfileEntry *pEntry; + ZipfileEntry *pNext; + + if( pTab->pWriteFd ){ + fclose(pTab->pWriteFd); + pTab->pWriteFd = 0; + } + for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){ + pNext = pEntry->pNext; + zipfileEntryFree(pEntry); + } + pTab->pFirstEntry = 0; + pTab->pLastEntry = 0; + pTab->szCurrent = 0; + pTab->szOrig = 0; +} + +/* ** This method is the destructor for zipfile vtab objects. */ static int zipfileDisconnect(sqlite3_vtab *pVtab){ + zipfileCleanupTransaction((ZipfileTab*)pVtab); sqlite3_free(pVtab); return SQLITE_OK; } @@ -4336,12 +4646,20 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ ** by zipfileOpen(). */ static void zipfileResetCursor(ZipfileCsr *pCsr){ - sqlite3_free(pCsr->cds.zFile); - pCsr->cds.zFile = 0; + ZipfileEntry *p; + ZipfileEntry *pNext; + pCsr->bEof = 0; if( pCsr->pFile ){ fclose(pCsr->pFile); pCsr->pFile = 0; + zipfileEntryFree(pCsr->pCurrent); + pCsr->pCurrent = 0; + } + + for(p=pCsr->pFreeEntry; p; p=pNext){ + pNext = p->pNext; + zipfileEntryFree(p); } } @@ -4355,12 +4673,8 @@ static int zipfileClose(sqlite3_vtab_cursor *cur){ zipfileResetCursor(pCsr); /* Remove this cursor from the ZipfileTab.pCsrList list. */ - for(pp=&pTab->pCsrList; *pp; pp=&((*pp)->pCsrNext)){ - if( *pp==pCsr ){ - *pp = pCsr->pCsrNext; - break; - } - } + for(pp=&pTab->pCsrList; *pp!=pCsr; pp=&((*pp)->pCsrNext)); + *pp = pCsr->pCsrNext; sqlite3_free(pCsr); return SQLITE_OK; @@ -4370,13 +4684,31 @@ static int zipfileClose(sqlite3_vtab_cursor *cur){ ** Set the error message for the virtual table associated with cursor ** pCsr to the results of vprintf(zFmt, ...). */ -static void zipfileSetErrmsg(ZipfileCsr *pCsr, const char *zFmt, ...){ +static void zipfileTableErr(ZipfileTab *pTab, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + sqlite3_free(pTab->base.zErrMsg); + pTab->base.zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} +static void zipfileCursorErr(ZipfileCsr *pCsr, const char *zFmt, ...){ va_list ap; va_start(ap, zFmt); + sqlite3_free(pCsr->base.pVtab->zErrMsg); pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); va_end(ap); } +/* +** Read nRead bytes of data from offset iOff of file pFile into buffer +** aRead[]. Return SQLITE_OK if successful, or an SQLite error code +** otherwise. +** +** If an error does occur, output variable (*pzErrmsg) may be set to point +** to an English language error message. It is the responsibility of the +** caller to eventually free this buffer using +** sqlite3_free(). +*/ static int zipfileReadData( FILE *pFile, /* Read from this file */ u8 *aRead, /* Read into this buffer */ @@ -4410,9 +4742,16 @@ static int zipfileAppendData( return SQLITE_OK; } +/* +** Read and return a 16-bit little-endian unsigned integer from buffer aBuf. +*/ static u16 zipfileGetU16(const u8 *aBuf){ return (aBuf[1] << 8) + aBuf[0]; } + +/* +** Read and return a 32-bit little-endian unsigned integer from buffer aBuf. +*/ static u32 zipfileGetU32(const u8 *aBuf){ return ((u32)(aBuf[3]) << 24) + ((u32)(aBuf[2]) << 16) @@ -4420,10 +4759,17 @@ static u32 zipfileGetU32(const u8 *aBuf){ + ((u32)(aBuf[0]) << 0); } +/* +** Write a 16-bit little endiate integer into buffer aBuf. +*/ static void zipfilePutU16(u8 *aBuf, u16 val){ aBuf[0] = val & 0xFF; aBuf[1] = (val>>8) & 0xFF; } + +/* +** Write a 32-bit little endiate integer into buffer aBuf. +*/ static void zipfilePutU32(u8 *aBuf, u32 val){ aBuf[0] = val & 0xFF; aBuf[1] = (val>>8) & 0xFF; @@ -4437,15 +4783,11 @@ static void zipfilePutU32(u8 *aBuf, u32 val){ #define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; } #define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; } -static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){ - return ((ZipfileTab*)(pCsr->base.pVtab))->aBuffer; -} - /* ** Magic numbers used to read CDS records. */ -#define ZIPFILE_CDS_FIXED_SZ 46 #define ZIPFILE_CDS_NFILE_OFF 28 +#define ZIPFILE_CDS_SZCOMPRESSED_OFF 20 /* ** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR @@ -4482,216 +4824,321 @@ static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){ } /* -** Read the CDS record for the current entry from disk into pCsr->cds. +** Decode the LFH record in buffer aBuf into (*pLFH). Return SQLITE_ERROR +** if the record is not well-formed, or SQLITE_OK otherwise. */ -static int zipfileCsrReadCDS(ZipfileCsr *pCsr){ - char **pzErr = &pCsr->base.pVtab->zErrMsg; - u8 *aRead; +static int zipfileReadLFH( + u8 *aBuffer, + ZipfileLFH *pLFH +){ + u8 *aRead = aBuffer; int rc = SQLITE_OK; - sqlite3_free(pCsr->cds.zFile); - pCsr->cds.zFile = 0; - - if( pCsr->pCurrent==0 ){ - aRead = zipfileCsrBuffer(pCsr); - rc = zipfileReadData( - pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr - ); + u32 sig = zipfileRead32(aRead); + if( sig!=ZIPFILE_SIGNATURE_LFH ){ + rc = SQLITE_ERROR; }else{ - aRead = pCsr->pCurrent->aCdsEntry; + pLFH->iVersionExtract = zipfileRead16(aRead); + pLFH->flags = zipfileRead16(aRead); + pLFH->iCompression = zipfileRead16(aRead); + pLFH->mTime = zipfileRead16(aRead); + pLFH->mDate = zipfileRead16(aRead); + pLFH->crc32 = zipfileRead32(aRead); + pLFH->szCompressed = zipfileRead32(aRead); + pLFH->szUncompressed = zipfileRead32(aRead); + pLFH->nFile = zipfileRead16(aRead); + pLFH->nExtra = zipfileRead16(aRead); } + return rc; +} - if( rc==SQLITE_OK ){ - rc = zipfileReadCDS(aRead, &pCsr->cds); - if( rc!=SQLITE_OK ){ - assert( pCsr->pCurrent==0 ); - zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff); - }else{ - int nRead; - if( pCsr->pCurrent==0 ){ - nRead = pCsr->cds.nFile + pCsr->cds.nExtra; - aRead = zipfileCsrBuffer(pCsr); - pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; - rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr); - }else{ - aRead = &aRead[ZIPFILE_CDS_FIXED_SZ]; - } - - if( rc==SQLITE_OK ){ - pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead); - pCsr->iNextOff += pCsr->cds.nFile; - pCsr->iNextOff += pCsr->cds.nExtra; - pCsr->iNextOff += pCsr->cds.nComment; - } - - /* Scan the cds.nExtra bytes of "extra" fields for any that can - ** be interpreted. The general format of an extra field is: - ** - ** Header ID 2 bytes - ** Data Size 2 bytes - ** Data N bytes - ** - */ - if( rc==SQLITE_OK ){ - u8 *p = &aRead[pCsr->cds.nFile]; - u8 *pEnd = &p[pCsr->cds.nExtra]; - - while( p<pEnd ){ - u16 id = zipfileRead16(p); - u16 nByte = zipfileRead16(p); - - switch( id ){ - case ZIPFILE_EXTRA_TIMESTAMP: { - u8 b = p[0]; - if( b & 0x01 ){ /* 0x01 -> modtime is present */ - pCsr->mTime = zipfileGetU32(&p[1]); - pCsr->flags |= ZIPFILE_MTIME_VALID; - } - break; - } - } - p += nByte; +/* +** Buffer aExtra (size nExtra bytes) contains zip archive "extra" fields. +** Scan through this buffer to find an "extra-timestamp" field. If one +** exists, extract the 32-bit modification-timestamp from it and store +** the value in output parameter *pmTime. +** +** Zero is returned if no extra-timestamp record could be found (and so +** *pmTime is left unchanged), or non-zero otherwise. +** +** The general format of an extra field is: +** +** Header ID 2 bytes +** Data Size 2 bytes +** Data N bytes +*/ +static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ + int ret = 0; + u8 *p = aExtra; + u8 *pEnd = &aExtra[nExtra]; + + while( p<pEnd ){ + u16 id = zipfileRead16(p); + u16 nByte = zipfileRead16(p); + + switch( id ){ + case ZIPFILE_EXTRA_TIMESTAMP: { + u8 b = p[0]; + if( b & 0x01 ){ /* 0x01 -> modtime is present */ + *pmTime = zipfileGetU32(&p[1]); + ret = 1; } + break; } } + + p += nByte; } + return ret; +} - return rc; +/* +** Convert the standard MS-DOS timestamp stored in the mTime and mDate +** fields of the CDS structure passed as the only argument to a 32-bit +** UNIX seconds-since-the-epoch timestamp. Return the result. +** +** "Standard" MS-DOS time format: +** +** File modification time: +** Bits 00-04: seconds divided by 2 +** Bits 05-10: minute +** Bits 11-15: hour +** File modification date: +** Bits 00-04: day +** Bits 05-08: month (1-12) +** Bits 09-15: years from 1980 +** +** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx +*/ +static u32 zipfileMtime(ZipfileCDS *pCDS){ + int Y = (1980 + ((pCDS->mDate >> 9) & 0x7F)); + int M = ((pCDS->mDate >> 5) & 0x0F); + int D = (pCDS->mDate & 0x1F); + int B = -13; + + int sec = (pCDS->mTime & 0x1F)*2; + int min = (pCDS->mTime >> 5) & 0x3F; + int hr = (pCDS->mTime >> 11) & 0x1F; + i64 JD; + + /* JD = INT(365.25 * (Y+4716)) + INT(30.6001 * (M+1)) + D + B - 1524.5 */ + + /* Calculate the JD in seconds for noon on the day in question */ + if( M<3 ){ + Y = Y-1; + M = M+12; + } + JD = (i64)(24*60*60) * ( + (int)(365.25 * (Y + 4716)) + + (int)(30.6001 * (M + 1)) + + D + B - 1524 + ); + + /* Correct the JD for the time within the day */ + JD += (hr-12) * 3600 + min * 60 + sec; + + /* Convert JD to unix timestamp (the JD epoch is 2440587.5) */ + return (u32)(JD - (i64)(24405875) * 24*60*6); } -static FILE *zipfileGetFd(ZipfileCsr *pCsr){ - if( pCsr->pFile ) return pCsr->pFile; - return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; +/* +** The opposite of zipfileMtime(). This function populates the mTime and +** mDate fields of the CDS structure passed as the first argument according +** to the UNIX timestamp value passed as the second. +*/ +static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ + /* Convert unix timestamp to JD (2440588 is noon on 1/1/1970) */ + i64 JD = (i64)2440588 + mUnixTime / (24*60*60); + + int A, B, C, D, E; + int yr, mon, day; + int hr, min, sec; + + A = (int)((JD - 1867216.25)/36524.25); + A = (int)(JD + 1 + A - (A/4)); + B = A + 1524; + C = (int)((B - 122.1)/365.25); + D = (36525*(C&32767))/100; + E = (int)((B-D)/30.6001); + + day = B - D - (int)(30.6001*E); + mon = (E<14 ? E-1 : E-13); + yr = mon>2 ? C-4716 : C-4715; + + hr = (mUnixTime % (24*60*60)) / (60*60); + min = (mUnixTime % (60*60)) / 60; + sec = (mUnixTime % 60); + + if( yr>=1980 ){ + pCds->mDate = (u16)(day + (mon << 5) + ((yr-1980) << 9)); + pCds->mTime = (u16)(sec/2 + (min<<5) + (hr<<11)); + }else{ + pCds->mDate = pCds->mTime = 0; + } + + assert( mUnixTime<315507600 + || mUnixTime==zipfileMtime(pCds) + || ((mUnixTime % 2) && mUnixTime-1==zipfileMtime(pCds)) + /* || (mUnixTime % 2) */ + ); } -static int zipfileReadLFH( - FILE *pFd, - i64 iOffset, - u8 *aTmp, - ZipfileLFH *pLFH, - char **pzErr +/* +** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in +** size) containing an entire zip archive image. Or, if aBlob is NULL, +** then pFile is a file-handle open on a zip file. In either case, this +** function creates a ZipfileEntry object based on the zip archive entry +** for which the CDS record is at offset iOff. +** +** If successful, SQLITE_OK is returned and (*ppEntry) set to point to +** the new object. Otherwise, an SQLite error code is returned and the +** final value of (*ppEntry) undefined. +*/ +static int zipfileGetEntry( + ZipfileTab *pTab, /* Store any error message here */ + const u8 *aBlob, /* Pointer to in-memory file image */ + int nBlob, /* Size of aBlob[] in bytes */ + FILE *pFile, /* If aBlob==0, read from this file */ + i64 iOff, /* Offset of CDS record */ + ZipfileEntry **ppEntry /* OUT: Pointer to new object */ ){ - u8 *aRead = aTmp; - static const int szFix = ZIPFILE_LFH_FIXED_SZ; - int rc; + u8 *aRead; + char **pzErr = &pTab->base.zErrMsg; + int rc = SQLITE_OK; + + if( aBlob==0 ){ + aRead = pTab->aBuffer; + rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); + }else{ + aRead = (u8*)&aBlob[iOff]; + } - rc = zipfileReadData(pFd, aRead, szFix, iOffset, pzErr); if( rc==SQLITE_OK ){ - u32 sig = zipfileRead32(aRead); - if( sig!=ZIPFILE_SIGNATURE_LFH ){ - *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", (int)iOffset); - rc = SQLITE_ERROR; + sqlite3_int64 nAlloc; + ZipfileEntry *pNew; + + int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]); + int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]); + nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]); + + nAlloc = sizeof(ZipfileEntry) + nExtra; + if( aBlob ){ + nAlloc += zipfileGetU32(&aRead[ZIPFILE_CDS_SZCOMPRESSED_OFF]); + } + + pNew = (ZipfileEntry*)sqlite3_malloc64(nAlloc); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(ZipfileEntry)); + rc = zipfileReadCDS(aRead, &pNew->cds); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); + }else if( aBlob==0 ){ + rc = zipfileReadData( + pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr + ); + }else{ + aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ]; + } + } + + if( rc==SQLITE_OK ){ + u32 *pt = &pNew->mUnixTime; + pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); + pNew->aExtra = (u8*)&pNew[1]; + memcpy(pNew->aExtra, &aRead[nFile], nExtra); + if( pNew->cds.zFile==0 ){ + rc = SQLITE_NOMEM; + }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){ + pNew->mUnixTime = zipfileMtime(&pNew->cds); + } + } + + if( rc==SQLITE_OK ){ + static const int szFix = ZIPFILE_LFH_FIXED_SZ; + ZipfileLFH lfh; + if( pFile ){ + rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr); + }else{ + aRead = (u8*)&aBlob[pNew->cds.iOffset]; + } + + rc = zipfileReadLFH(aRead, &lfh); + if( rc==SQLITE_OK ){ + pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; + pNew->iDataOff += lfh.nFile + lfh.nExtra; + if( aBlob && pNew->cds.szCompressed ){ + pNew->aData = &pNew->aExtra[nExtra]; + memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); + } + }else{ + *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", + (int)pNew->cds.iOffset + ); + } + } + + if( rc!=SQLITE_OK ){ + zipfileEntryFree(pNew); }else{ - pLFH->iVersionExtract = zipfileRead16(aRead); - pLFH->flags = zipfileRead16(aRead); - pLFH->iCompression = zipfileRead16(aRead); - pLFH->mTime = zipfileRead16(aRead); - pLFH->mDate = zipfileRead16(aRead); - pLFH->crc32 = zipfileRead32(aRead); - pLFH->szCompressed = zipfileRead32(aRead); - pLFH->szUncompressed = zipfileRead32(aRead); - pLFH->nFile = zipfileRead16(aRead); - pLFH->nExtra = zipfileRead16(aRead); - assert( aRead==&aTmp[szFix] ); + *ppEntry = pNew; } } - return rc; -} -static int zipfileCsrReadLFH(ZipfileCsr *pCsr){ - FILE *pFile = zipfileGetFd(pCsr); - char **pzErr = &pCsr->base.pVtab->zErrMsg; - u8 *aRead = zipfileCsrBuffer(pCsr); - int rc = zipfileReadLFH(pFile, pCsr->cds.iOffset, aRead, &pCsr->lfh, pzErr); - pCsr->iDataOff = pCsr->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; - pCsr->iDataOff += pCsr->lfh.nFile+pCsr->lfh.nExtra; return rc; } - /* ** Advance an ZipfileCsr to its next row of output. */ static int zipfileNext(sqlite3_vtab_cursor *cur){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; int rc = SQLITE_OK; - pCsr->flags = 0; - if( pCsr->pCurrent==0 ){ + if( pCsr->pFile ){ i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; + zipfileEntryFree(pCsr->pCurrent); + pCsr->pCurrent = 0; if( pCsr->iNextOff>=iEof ){ pCsr->bEof = 1; + }else{ + ZipfileEntry *p = 0; + ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab); + rc = zipfileGetEntry(pTab, 0, 0, pCsr->pFile, pCsr->iNextOff, &p); + if( rc==SQLITE_OK ){ + pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; + pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment; + } + pCsr->pCurrent = p; } }else{ - assert( pCsr->pFile==0 ); - do { + if( !pCsr->bNoop ){ pCsr->pCurrent = pCsr->pCurrent->pNext; - }while( pCsr->pCurrent && pCsr->pCurrent->bDeleted ); + } if( pCsr->pCurrent==0 ){ pCsr->bEof = 1; } } - if( pCsr->bEof==0 ){ - rc = zipfileCsrReadCDS(pCsr); - if( rc==SQLITE_OK ){ - rc = zipfileCsrReadLFH(pCsr); - } - } - + pCsr->bNoop = 0; return rc; } +static void zipfileFree(void *p) { + sqlite3_free(p); +} + /* -** "Standard" MS-DOS time format: +** Buffer aIn (size nIn bytes) contains compressed data. Uncompressed, the +** size is nOut bytes. This function uncompresses the data and sets the +** return value in context pCtx to the result (a blob). ** -** File modification time: -** Bits 00-04: seconds divided by 2 -** Bits 05-10: minute -** Bits 11-15: hour -** File modification date: -** Bits 00-04: day -** Bits 05-08: month (1-12) -** Bits 09-15: years from 1980 +** If an error occurs, an error code is left in pCtx instead. */ -static time_t zipfileMtime(ZipfileCsr *pCsr){ - struct tm t; - memset(&t, 0, sizeof(t)); - t.tm_sec = (pCsr->cds.mTime & 0x1F)*2; - t.tm_min = (pCsr->cds.mTime >> 5) & 0x2F; - t.tm_hour = (pCsr->cds.mTime >> 11) & 0x1F; - - t.tm_mday = (pCsr->cds.mDate & 0x1F); - t.tm_mon = ((pCsr->cds.mDate >> 5) & 0x0F) - 1; - t.tm_year = 80 + ((pCsr->cds.mDate >> 9) & 0x7F); - - return mktime(&t); -} - -static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){ - time_t t = (time_t)mTime; - struct tm res; - -#if !defined(_WIN32) && !defined(WIN32) - localtime_r(&t, &res); -#else - memcpy(&res, localtime(&t), sizeof(struct tm)); -#endif - - pCds->mTime = (u16)( - (res.tm_sec / 2) + - (res.tm_min << 5) + - (res.tm_hour << 11)); - - pCds->mDate = (u16)( - (res.tm_mday-1) + - ((res.tm_mon+1) << 5) + - ((res.tm_year-80) << 9)); -} - static void zipfileInflate( - sqlite3_context *pCtx, /* Store error here, if any */ + sqlite3_context *pCtx, /* Store result here */ const u8 *aIn, /* Compressed data */ int nIn, /* Size of buffer aIn[] in bytes */ int nOut /* Expected output size */ @@ -4717,7 +5164,8 @@ static void zipfileInflate( if( err!=Z_STREAM_END ){ zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err); }else{ - sqlite3_result_blob(pCtx, aRes, nOut, SQLITE_TRANSIENT); + sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree); + aRes = 0; } } sqlite3_free(aRes); @@ -4725,16 +5173,28 @@ static void zipfileInflate( } } +/* +** Buffer aIn (size nIn bytes) contains uncompressed data. This function +** compresses it and sets (*ppOut) to point to a buffer containing the +** compressed data. The caller is responsible for eventually calling +** sqlite3_free() to release buffer (*ppOut). Before returning, (*pnOut) +** is set to the size of buffer (*ppOut) in bytes. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error +** code is returned and an error message left in virtual-table handle +** pTab. The values of (*ppOut) and (*pnOut) are left unchanged in this +** case. +*/ static int zipfileDeflate( - ZipfileTab *pTab, /* Set error message here */ const u8 *aIn, int nIn, /* Input */ - u8 **ppOut, int *pnOut /* Output */ + u8 **ppOut, int *pnOut, /* Output */ + char **pzErr /* OUT: Error message */ ){ - int nAlloc = (int)compressBound(nIn); + sqlite3_int64 nAlloc = compressBound(nIn); u8 *aOut; int rc = SQLITE_OK; - aOut = (u8*)sqlite3_malloc(nAlloc); + aOut = (u8*)sqlite3_malloc64(nAlloc); if( aOut==0 ){ rc = SQLITE_NOMEM; }else{ @@ -4754,7 +5214,7 @@ static int zipfileDeflate( *pnOut = (int)str.total_out; }else{ sqlite3_free(aOut); - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error"); + *pzErr = sqlite3_mprintf("zipfile: deflate() error"); rc = SQLITE_ERROR; } deflateEnd(&str); @@ -4774,60 +5234,66 @@ static int zipfileColumn( int i /* Which column to return */ ){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; + ZipfileCDS *pCDS = &pCsr->pCurrent->cds; int rc = SQLITE_OK; switch( i ){ case 0: /* name */ - sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT); + sqlite3_result_text(ctx, pCDS->zFile, -1, SQLITE_TRANSIENT); break; case 1: /* mode */ /* TODO: Whether or not the following is correct surely depends on ** the platform on which the archive was created. */ - sqlite3_result_int(ctx, pCsr->cds.iExternalAttr >> 16); + sqlite3_result_int(ctx, pCDS->iExternalAttr >> 16); break; case 2: { /* mtime */ - if( pCsr->flags & ZIPFILE_MTIME_VALID ){ - sqlite3_result_int64(ctx, pCsr->mTime); - }else{ - sqlite3_result_int64(ctx, zipfileMtime(pCsr)); - } + sqlite3_result_int64(ctx, pCsr->pCurrent->mUnixTime); break; } case 3: { /* sz */ if( sqlite3_vtab_nochange(ctx)==0 ){ - sqlite3_result_int64(ctx, pCsr->cds.szUncompressed); + sqlite3_result_int64(ctx, pCDS->szUncompressed); } break; } case 4: /* rawdata */ if( sqlite3_vtab_nochange(ctx) ) break; case 5: { /* data */ - if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){ - int sz = pCsr->cds.szCompressed; - int szFinal = pCsr->cds.szUncompressed; + if( i==4 || pCDS->iCompression==0 || pCDS->iCompression==8 ){ + int sz = pCDS->szCompressed; + int szFinal = pCDS->szUncompressed; if( szFinal>0 ){ - u8 *aBuf = sqlite3_malloc(sz); - if( aBuf==0 ){ - rc = SQLITE_NOMEM; + u8 *aBuf; + u8 *aFree = 0; + if( pCsr->pCurrent->aData ){ + aBuf = pCsr->pCurrent->aData; }else{ - FILE *pFile = zipfileGetFd(pCsr); - rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff, - &pCsr->base.pVtab->zErrMsg - ); + aBuf = aFree = sqlite3_malloc64(sz); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + FILE *pFile = pCsr->pFile; + if( pFile==0 ){ + pFile = ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; + } + rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff, + &pCsr->base.pVtab->zErrMsg + ); + } } if( rc==SQLITE_OK ){ - if( i==5 && pCsr->cds.iCompression ){ + if( i==5 && pCDS->iCompression ){ zipfileInflate(ctx, aBuf, sz, szFinal); }else{ sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); } - sqlite3_free(aBuf); } + sqlite3_free(aFree); }else{ /* Figure out if this is a directory or a zero-sized file. Consider ** it to be a directory either if the mode suggests so, or if ** the final character in the name is '/'. */ - u32 mode = pCsr->cds.iExternalAttr >> 16; - if( !(mode & S_IFDIR) && pCsr->cds.zFile[pCsr->cds.nFile-1]!='/' ){ + u32 mode = pCDS->iExternalAttr >> 16; + if( !(mode & S_IFDIR) && pCDS->zFile[pCDS->nFile-1]!='/' ){ sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC); } } @@ -4835,9 +5301,10 @@ static int zipfileColumn( break; } case 6: /* method */ - sqlite3_result_int(ctx, pCsr->cds.iCompression); + sqlite3_result_int(ctx, pCDS->iCompression); break; - case 7: /* z */ + default: /* z */ + assert( i==7 ); sqlite3_result_int64(ctx, pCsr->iId); break; } @@ -4846,16 +5313,7 @@ static int zipfileColumn( } /* -** Return the rowid for the current row. -*/ -static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - assert( 0 ); - return SQLITE_OK; -} - -/* -** Return TRUE if the cursor has been moved off of the last -** row of output. +** Return TRUE if the cursor is at EOF. */ static int zipfileEof(sqlite3_vtab_cursor *cur){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; @@ -4863,28 +5321,43 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){ } /* +** If aBlob is not NULL, then it points to a buffer nBlob bytes in size +** containing an entire zip archive image. Or, if aBlob is NULL, then pFile +** is guaranteed to be a file-handle open on a zip file. +** +** This function attempts to locate the EOCD record within the zip archive +** and populate *pEOCD with the results of decoding it. SQLITE_OK is +** returned if successful. Otherwise, an SQLite error code is returned and +** an English language error message may be left in virtual-table pTab. */ static int zipfileReadEOCD( ZipfileTab *pTab, /* Return errors here */ - FILE *pFile, /* Read from this file */ + const u8 *aBlob, /* Pointer to in-memory file image */ + int nBlob, /* Size of aBlob[] in bytes */ + FILE *pFile, /* Read from this file if aBlob==0 */ ZipfileEOCD *pEOCD /* Object to populate */ ){ u8 *aRead = pTab->aBuffer; /* Temporary buffer */ - i64 szFile; /* Total size of file in bytes */ int nRead; /* Bytes to read from file */ - i64 iOff; /* Offset to read from */ - int rc; + int rc = SQLITE_OK; - fseek(pFile, 0, SEEK_END); - szFile = (i64)ftell(pFile); - if( szFile==0 ){ - memset(pEOCD, 0, sizeof(ZipfileEOCD)); - return SQLITE_OK; + if( aBlob==0 ){ + i64 iOff; /* Offset to read from */ + i64 szFile; /* Total size of file in bytes */ + fseek(pFile, 0, SEEK_END); + szFile = (i64)ftell(pFile); + if( szFile==0 ){ + memset(pEOCD, 0, sizeof(ZipfileEOCD)); + return SQLITE_OK; + } + nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); + iOff = szFile - nRead; + rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); + }else{ + nRead = (int)(MIN(nBlob, ZIPFILE_BUFFER_SIZE)); + aRead = (u8*)&aBlob[nBlob-nRead]; } - nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); - iOff = szFile - nRead; - rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); if( rc==SQLITE_OK ){ int i; @@ -4910,17 +5383,59 @@ static int zipfileReadEOCD( pEOCD->nEntryTotal = zipfileRead16(aRead); pEOCD->nSize = zipfileRead32(aRead); pEOCD->iOffset = zipfileRead32(aRead); + } -#if 0 - printf("iDisk=%d iFirstDisk=%d nEntry=%d " - "nEntryTotal=%d nSize=%d iOffset=%d", - (int)pEOCD->iDisk, (int)pEOCD->iFirstDisk, (int)pEOCD->nEntry, - (int)pEOCD->nEntryTotal, (int)pEOCD->nSize, (int)pEOCD->iOffset - ); -#endif + return rc; +} + +/* +** Add object pNew to the linked list that begins at ZipfileTab.pFirstEntry +** and ends with pLastEntry. If argument pBefore is NULL, then pNew is added +** to the end of the list. Otherwise, it is added to the list immediately +** before pBefore (which is guaranteed to be a part of said list). +*/ +static void zipfileAddEntry( + ZipfileTab *pTab, + ZipfileEntry *pBefore, + ZipfileEntry *pNew +){ + assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); + assert( pNew->pNext==0 ); + if( pBefore==0 ){ + if( pTab->pFirstEntry==0 ){ + pTab->pFirstEntry = pTab->pLastEntry = pNew; + }else{ + assert( pTab->pLastEntry->pNext==0 ); + pTab->pLastEntry->pNext = pNew; + pTab->pLastEntry = pNew; + } + }else{ + ZipfileEntry **pp; + for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); + pNew->pNext = pBefore; + *pp = pNew; } +} - return SQLITE_OK; +static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){ + ZipfileEOCD eocd; + int rc; + int i; + i64 iOff; + + rc = zipfileReadEOCD(pTab, aBlob, nBlob, pTab->pWriteFd, &eocd); + iOff = eocd.iOffset; + for(i=0; rc==SQLITE_OK && i<eocd.nEntry; i++){ + ZipfileEntry *pNew = 0; + rc = zipfileGetEntry(pTab, aBlob, nBlob, pTab->pWriteFd, iOff, &pNew); + + if( rc==SQLITE_OK ){ + zipfileAddEntry(pTab, 0, pNew); + iOff += ZIPFILE_CDS_FIXED_SZ; + iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment; + } + } + return rc; } /* @@ -4933,29 +5448,37 @@ static int zipfileFilter( ){ ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; ZipfileCsr *pCsr = (ZipfileCsr*)cur; - const char *zFile; /* Zip file to scan */ + const char *zFile = 0; /* Zip file to scan */ int rc = SQLITE_OK; /* Return Code */ + int bInMemory = 0; /* True for an in-memory zipfile */ zipfileResetCursor(pCsr); if( pTab->zFile ){ zFile = pTab->zFile; }else if( idxNum==0 ){ - /* Error. This is an eponymous virtual table and the user has not - ** supplied a file name. */ - zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument"); + zipfileCursorErr(pCsr, "zipfile() function requires an argument"); return SQLITE_ERROR; + }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ + const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); + int nBlob = sqlite3_value_bytes(argv[0]); + assert( pTab->pFirstEntry==0 ); + rc = zipfileLoadDirectory(pTab, aBlob, nBlob); + pCsr->pFreeEntry = pTab->pFirstEntry; + pTab->pFirstEntry = pTab->pLastEntry = 0; + if( rc!=SQLITE_OK ) return rc; + bInMemory = 1; }else{ zFile = (const char*)sqlite3_value_text(argv[0]); } - if( pTab->pWriteFd==0 ){ + if( 0==pTab->pWriteFd && 0==bInMemory ){ pCsr->pFile = fopen(zFile, "rb"); if( pCsr->pFile==0 ){ - zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile); + zipfileCursorErr(pCsr, "cannot open file: %s", zFile); rc = SQLITE_ERROR; }else{ - rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd); + rc = zipfileReadEOCD(pTab, 0, 0, pCsr->pFile, &pCsr->eocd); if( rc==SQLITE_OK ){ if( pCsr->eocd.nEntry==0 ){ pCsr->bEof = 1; @@ -4966,12 +5489,9 @@ static int zipfileFilter( } } }else{ - ZipfileEntry e; - memset(&e, 0, sizeof(e)); - e.pNext = pTab->pFirstEntry; - pCsr->pCurrent = &e; + pCsr->bNoop = 1; + pCsr->pCurrent = pCsr->pFreeEntry ? pCsr->pFreeEntry : pTab->pFirstEntry; rc = zipfileNext(cur); - assert( pCsr->pCurrent!=&e ); } return rc; @@ -4985,204 +5505,90 @@ static int zipfileBestIndex( sqlite3_index_info *pIdxInfo ){ int i; + int idx = -1; + int unusable = 0; for(i=0; i<pIdxInfo->nConstraint; i++){ const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; - if( pCons->usable==0 ) continue; - if( pCons->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue; - break; + if( pCons->usable==0 ){ + unusable = 1; + }else if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + idx = i; + } } - - if( i<pIdxInfo->nConstraint ){ - pIdxInfo->aConstraintUsage[i].argvIndex = 1; - pIdxInfo->aConstraintUsage[i].omit = 1; + if( idx>=0 ){ + pIdxInfo->aConstraintUsage[idx].argvIndex = 1; + pIdxInfo->aConstraintUsage[idx].omit = 1; pIdxInfo->estimatedCost = 1000.0; pIdxInfo->idxNum = 1; - }else{ - pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50); - pIdxInfo->idxNum = 0; + }else if( unusable ){ + return SQLITE_CONSTRAINT; } - return SQLITE_OK; } -/* -** Add object pNew to the end of the linked list that begins at -** ZipfileTab.pFirstEntry and ends with pLastEntry. -*/ -static void zipfileAddEntry( - ZipfileTab *pTab, - ZipfileEntry *pBefore, - ZipfileEntry *pNew -){ - assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); - assert( pNew->pNext==0 ); - if( pBefore==0 ){ - if( pTab->pFirstEntry==0 ){ - pTab->pFirstEntry = pTab->pLastEntry = pNew; - }else{ - assert( pTab->pLastEntry->pNext==0 ); - pTab->pLastEntry->pNext = pNew; - pTab->pLastEntry = pNew; +static ZipfileEntry *zipfileNewEntry(const char *zPath){ + ZipfileEntry *pNew; + pNew = sqlite3_malloc(sizeof(ZipfileEntry)); + if( pNew ){ + memset(pNew, 0, sizeof(ZipfileEntry)); + pNew->cds.zFile = sqlite3_mprintf("%s", zPath); + if( pNew->cds.zFile==0 ){ + sqlite3_free(pNew); + pNew = 0; } - }else{ - ZipfileEntry **pp; - for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); - pNew->pNext = pBefore; - *pp = pNew; } + return pNew; } -static int zipfileLoadDirectory(ZipfileTab *pTab){ - ZipfileEOCD eocd; - int rc; - - rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd); - if( rc==SQLITE_OK && eocd.nEntry>0 ){ - int i; - int iOff = 0; - u8 *aBuf = sqlite3_malloc(eocd.nSize); - if( aBuf==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = zipfileReadData( - pTab->pWriteFd, aBuf, eocd.nSize, eocd.iOffset, &pTab->base.zErrMsg - ); - } - - for(i=0; rc==SQLITE_OK && i<eocd.nEntry; i++){ - u16 nFile; - u16 nExtra; - u16 nComment; - ZipfileEntry *pNew; - u8 *aRec = &aBuf[iOff]; - - nFile = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF]); - nExtra = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF+2]); - nComment = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF+4]); - - pNew = sqlite3_malloc( - sizeof(ZipfileEntry) - + nFile+1 - + ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment - ); - if( pNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(pNew, 0, sizeof(ZipfileEntry)); - pNew->zPath = (char*)&pNew[1]; - memcpy(pNew->zPath, &aRec[ZIPFILE_CDS_FIXED_SZ], nFile); - pNew->zPath[nFile] = '\0'; - pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1]; - pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; - memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry); - zipfileAddEntry(pTab, 0, pNew); - } - - iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; - } +static int zipfileSerializeLFH(ZipfileEntry *pEntry, u8 *aBuf){ + ZipfileCDS *pCds = &pEntry->cds; + u8 *a = aBuf; - sqlite3_free(aBuf); - } + pCds->nExtra = 9; - return rc; -} + /* Write the LFH itself */ + zipfileWrite32(a, ZIPFILE_SIGNATURE_LFH); + zipfileWrite16(a, pCds->iVersionExtract); + zipfileWrite16(a, pCds->flags); + zipfileWrite16(a, pCds->iCompression); + zipfileWrite16(a, pCds->mTime); + zipfileWrite16(a, pCds->mDate); + zipfileWrite32(a, pCds->crc32); + zipfileWrite32(a, pCds->szCompressed); + zipfileWrite32(a, pCds->szUncompressed); + zipfileWrite16(a, (u16)pCds->nFile); + zipfileWrite16(a, pCds->nExtra); + assert( a==&aBuf[ZIPFILE_LFH_FIXED_SZ] ); -static ZipfileEntry *zipfileNewEntry( - ZipfileCDS *pCds, /* Values for fixed size part of CDS */ - const char *zPath, /* Path for new entry */ - int nPath, /* strlen(zPath) */ - u32 mTime /* Modification time (or 0) */ -){ - u8 *aWrite; - ZipfileEntry *pNew; - pCds->nFile = (u16)nPath; - pCds->nExtra = mTime ? 9 : 0; - pNew = (ZipfileEntry*)sqlite3_malloc( - sizeof(ZipfileEntry) + - nPath+1 + - ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra - ); + /* Add the file name */ + memcpy(a, pCds->zFile, (int)pCds->nFile); + a += (int)pCds->nFile; - if( pNew ){ - memset(pNew, 0, sizeof(ZipfileEntry)); - pNew->zPath = (char*)&pNew[1]; - pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1]; - pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra; - memcpy(pNew->zPath, zPath, nPath+1); - - aWrite = pNew->aCdsEntry; - zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS); - zipfileWrite16(aWrite, pCds->iVersionMadeBy); - zipfileWrite16(aWrite, pCds->iVersionExtract); - zipfileWrite16(aWrite, pCds->flags); - zipfileWrite16(aWrite, pCds->iCompression); - zipfileWrite16(aWrite, pCds->mTime); - zipfileWrite16(aWrite, pCds->mDate); - zipfileWrite32(aWrite, pCds->crc32); - zipfileWrite32(aWrite, pCds->szCompressed); - zipfileWrite32(aWrite, pCds->szUncompressed); - zipfileWrite16(aWrite, pCds->nFile); - zipfileWrite16(aWrite, pCds->nExtra); - zipfileWrite16(aWrite, pCds->nComment); assert( pCds->nComment==0 ); - zipfileWrite16(aWrite, pCds->iDiskStart); - zipfileWrite16(aWrite, pCds->iInternalAttr); - zipfileWrite32(aWrite, pCds->iExternalAttr); - zipfileWrite32(aWrite, pCds->iOffset); - assert( aWrite==&pNew->aCdsEntry[ZIPFILE_CDS_FIXED_SZ] ); - memcpy(aWrite, zPath, nPath); - if( pCds->nExtra ){ - aWrite += nPath; - zipfileWrite16(aWrite, ZIPFILE_EXTRA_TIMESTAMP); - zipfileWrite16(aWrite, 5); - *aWrite++ = 0x01; - zipfileWrite32(aWrite, mTime); - } - } + /* The "extra" data */ + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); - return pNew; + return a-aBuf; } static int zipfileAppendEntry( ZipfileTab *pTab, - ZipfileCDS *pCds, - const char *zPath, /* Path for new entry */ - int nPath, /* strlen(zPath) */ + ZipfileEntry *pEntry, const u8 *pData, - int nData, - u32 mTime + int nData ){ u8 *aBuf = pTab->aBuffer; + int nBuf; int rc; - zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH); - zipfileWrite16(aBuf, pCds->iVersionExtract); - zipfileWrite16(aBuf, pCds->flags); - zipfileWrite16(aBuf, pCds->iCompression); - zipfileWrite16(aBuf, pCds->mTime); - zipfileWrite16(aBuf, pCds->mDate); - zipfileWrite32(aBuf, pCds->crc32); - zipfileWrite32(aBuf, pCds->szCompressed); - zipfileWrite32(aBuf, pCds->szUncompressed); - zipfileWrite16(aBuf, (u16)nPath); - zipfileWrite16(aBuf, pCds->nExtra); - assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] ); - rc = zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer)); - if( rc==SQLITE_OK ){ - rc = zipfileAppendData(pTab, (const u8*)zPath, nPath); - } - - if( rc==SQLITE_OK && pCds->nExtra ){ - aBuf = pTab->aBuffer; - zipfileWrite16(aBuf, ZIPFILE_EXTRA_TIMESTAMP); - zipfileWrite16(aBuf, 5); - *aBuf++ = 0x01; - zipfileWrite32(aBuf, mTime); - rc = zipfileAppendData(pTab, pTab->aBuffer, 9); - } - + nBuf = zipfileSerializeLFH(pEntry, aBuf); + rc = zipfileAppendData(pTab, aBuf, nBuf); if( rc==SQLITE_OK ){ + pEntry->iDataOff = pTab->szCurrent; rc = zipfileAppendData(pTab, pData, nData); } @@ -5190,15 +5596,15 @@ static int zipfileAppendEntry( } static int zipfileGetMode( - ZipfileTab *pTab, sqlite3_value *pVal, - u32 defaultMode, /* Value to use if pVal IS NULL */ - u32 *pMode + int bIsDir, /* If true, default to directory */ + u32 *pMode, /* OUT: Mode value */ + char **pzErr /* OUT: Error message */ ){ const char *z = (const char*)sqlite3_value_text(pVal); u32 mode = 0; if( z==0 ){ - mode = defaultMode; + mode = (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)); }else if( z[0]>='0' && z[0]<='9' ){ mode = (unsigned int)sqlite3_value_int(pVal); }else{ @@ -5208,9 +5614,7 @@ static int zipfileGetMode( switch( z[0] ){ case '-': mode |= S_IFREG; break; case 'd': mode |= S_IFDIR; break; -#if !defined(_WIN32) && !defined(WIN32) case 'l': mode |= S_IFLNK; break; -#endif default: goto parse_error; } for(i=1; i<10; i++){ @@ -5218,11 +5622,17 @@ static int zipfileGetMode( else if( z[i]!='-' ) goto parse_error; } } + if( ((mode & S_IFDIR)==0)==bIsDir ){ + /* The "mode" attribute is a directory, but data has been specified. + ** Or vice-versa - no data but "mode" is a file or symlink. */ + *pzErr = sqlite3_mprintf("zipfile: mode does not match data"); + return SQLITE_CONSTRAINT; + } *pMode = mode; return SQLITE_OK; parse_error: - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: parse error in mode: %s", z); + *pzErr = sqlite3_mprintf("zipfile: parse error in mode: %s", z); return SQLITE_ERROR; } @@ -5238,6 +5648,81 @@ static int zipfileComparePath(const char *zA, const char *zB, int nB){ return 1; } +static int zipfileBegin(sqlite3_vtab *pVtab){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; + + assert( pTab->pWriteFd==0 ); + + /* Open a write fd on the file. Also load the entire central directory + ** structure into memory. During the transaction any new file data is + ** appended to the archive file, but the central directory is accumulated + ** in main-memory until the transaction is committed. */ + pTab->pWriteFd = fopen(pTab->zFile, "ab+"); + if( pTab->pWriteFd==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: failed to open file %s for writing", pTab->zFile + ); + rc = SQLITE_ERROR; + }else{ + fseek(pTab->pWriteFd, 0, SEEK_END); + pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); + rc = zipfileLoadDirectory(pTab, 0, 0); + } + + if( rc!=SQLITE_OK ){ + zipfileCleanupTransaction(pTab); + } + + return rc; +} + +/* +** Return the current time as a 32-bit timestamp in UNIX epoch format (like +** time(2)). +*/ +static u32 zipfileTime(void){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + u32 ret; + if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ + i64 ms; + pVfs->xCurrentTimeInt64(pVfs, &ms); + ret = (u32)((ms/1000) - ((i64)24405875 * 8640)); + }else{ + double day; + pVfs->xCurrentTime(pVfs, &day); + ret = (u32)((day - 2440587.5) * 86400); + } + return ret; +} + +/* +** Return a 32-bit timestamp in UNIX epoch format. +** +** If the value passed as the only argument is either NULL or an SQL NULL, +** return the current time. Otherwise, return the value stored in (*pVal) +** cast to a 32-bit unsigned integer. +*/ +static u32 zipfileGetTime(sqlite3_value *pVal){ + if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){ + return zipfileTime(); + } + return (u32)sqlite3_value_int64(pVal); +} + +/* +** Unless it is NULL, entry pOld is currently part of the pTab->pFirstEntry +** linked list. Remove it from the list and free the object. +*/ +static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){ + if( pOld ){ + ZipfileEntry **pp; + for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext)); + *pp = (*pp)->pNext; + zipfileEntryFree(pOld); + } +} + /* ** xUpdate method. */ @@ -5252,7 +5737,7 @@ static int zipfileUpdate( ZipfileEntry *pNew = 0; /* New in-memory CDS entry */ u32 mode = 0; /* Mode for new entry */ - i64 mTime = 0; /* Modification time for new entry */ + u32 mTime = 0; /* Modification time for new entry */ i64 sz = 0; /* Uncompressed size */ const char *zPath = 0; /* Path for new entry */ int nPath = 0; /* strlen(zPath) */ @@ -5261,217 +5746,239 @@ static int zipfileUpdate( int iMethod = 0; /* Compression method for new entry */ u8 *pFree = 0; /* Free this */ char *zFree = 0; /* Also free this */ - ZipfileCDS cds; /* New Central Directory Structure entry */ ZipfileEntry *pOld = 0; + ZipfileEntry *pOld2 = 0; + int bUpdate = 0; /* True for an update that modifies "name" */ int bIsDir = 0; u32 iCrc32 = 0; - assert( pTab->zFile ); - assert( pTab->pWriteFd ); + if( pTab->pWriteFd==0 ){ + rc = zipfileBegin(pVtab); + if( rc!=SQLITE_OK ) return rc; + } + /* If this is a DELETE or UPDATE, find the archive entry to delete. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ const char *zDelete = (const char*)sqlite3_value_text(apVal[0]); int nDelete = (int)strlen(zDelete); + if( nVal>1 ){ + const char *zUpdate = (const char*)sqlite3_value_text(apVal[1]); + if( zUpdate && zipfileComparePath(zUpdate, zDelete, nDelete)!=0 ){ + bUpdate = 1; + } + } for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){ - if( pOld->bDeleted ) continue; - if( zipfileComparePath(pOld->zPath, zDelete, nDelete)==0 ){ - pOld->bDeleted = 1; + if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){ break; } assert( pOld->pNext ); } - if( nVal==1 ) return SQLITE_OK; } - /* Check that "sz" and "rawdata" are both NULL: */ - if( sqlite3_value_type(apVal[5])!=SQLITE_NULL - || sqlite3_value_type(apVal[6])!=SQLITE_NULL - ){ - rc = SQLITE_CONSTRAINT; - } + if( nVal>1 ){ + /* Check that "sz" and "rawdata" are both NULL: */ + if( sqlite3_value_type(apVal[5])!=SQLITE_NULL ){ + zipfileTableErr(pTab, "sz must be NULL"); + rc = SQLITE_CONSTRAINT; + } + if( sqlite3_value_type(apVal[6])!=SQLITE_NULL ){ + zipfileTableErr(pTab, "rawdata must be NULL"); + rc = SQLITE_CONSTRAINT; + } - if( rc==SQLITE_OK ){ - if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){ - /* data=NULL. A directory */ - bIsDir = 1; - }else{ - /* Value specified for "data", and possibly "method". This must be - ** a regular file or a symlink. */ - const u8 *aIn = sqlite3_value_blob(apVal[7]); - int nIn = sqlite3_value_bytes(apVal[7]); - int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; - - iMethod = sqlite3_value_int(apVal[8]); - sz = nIn; - pData = aIn; - nData = nIn; - if( iMethod!=0 && iMethod!=8 ){ - rc = SQLITE_CONSTRAINT; + if( rc==SQLITE_OK ){ + if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){ + /* data=NULL. A directory */ + bIsDir = 1; }else{ - if( bAuto || iMethod ){ - int nCmp; - rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp); - if( rc==SQLITE_OK ){ - if( iMethod || nCmp<nIn ){ - iMethod = 8; - pData = pFree; - nData = nCmp; + /* Value specified for "data", and possibly "method". This must be + ** a regular file or a symlink. */ + const u8 *aIn = sqlite3_value_blob(apVal[7]); + int nIn = sqlite3_value_bytes(apVal[7]); + int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; + + iMethod = sqlite3_value_int(apVal[8]); + sz = nIn; + pData = aIn; + nData = nIn; + if( iMethod!=0 && iMethod!=8 ){ + zipfileTableErr(pTab, "unknown compression method: %d", iMethod); + rc = SQLITE_CONSTRAINT; + }else{ + if( bAuto || iMethod ){ + int nCmp; + rc = zipfileDeflate(aIn, nIn, &pFree, &nCmp, &pTab->base.zErrMsg); + if( rc==SQLITE_OK ){ + if( iMethod || nCmp<nIn ){ + iMethod = 8; + pData = pFree; + nData = nCmp; + } } } + iCrc32 = crc32(0, aIn, nIn); } - iCrc32 = crc32(0, aIn, nIn); } } - } - if( rc==SQLITE_OK ){ - rc = zipfileGetMode(pTab, apVal[3], - (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode - ); - if( rc==SQLITE_OK && (bIsDir == ((mode & S_IFDIR)==0)) ){ - /* The "mode" attribute is a directory, but data has been specified. - ** Or vice-versa - no data but "mode" is a file or symlink. */ - rc = SQLITE_CONSTRAINT; + if( rc==SQLITE_OK ){ + rc = zipfileGetMode(apVal[3], bIsDir, &mode, &pTab->base.zErrMsg); } - } - if( rc==SQLITE_OK ){ - zPath = (const char*)sqlite3_value_text(apVal[2]); - nPath = (int)strlen(zPath); - if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){ - mTime = (sqlite3_int64)time(0); - }else{ - mTime = sqlite3_value_int64(apVal[4]); + if( rc==SQLITE_OK ){ + zPath = (const char*)sqlite3_value_text(apVal[2]); + nPath = (int)strlen(zPath); + mTime = zipfileGetTime(apVal[4]); + } + + if( rc==SQLITE_OK && bIsDir ){ + /* For a directory, check that the last character in the path is a + ** '/'. This appears to be required for compatibility with info-zip + ** (the unzip command on unix). It does not create directories + ** otherwise. */ + if( zPath[nPath-1]!='/' ){ + zFree = sqlite3_mprintf("%s/", zPath); + if( zFree==0 ){ rc = SQLITE_NOMEM; } + zPath = (const char*)zFree; + nPath++; + } } - } - if( rc==SQLITE_OK && bIsDir ){ - /* For a directory, check that the last character in the path is a - ** '/'. This appears to be required for compatibility with info-zip - ** (the unzip command on unix). It does not create directories - ** otherwise. */ - if( zPath[nPath-1]!='/' ){ - zFree = sqlite3_mprintf("%s/", zPath); - if( zFree==0 ){ rc = SQLITE_NOMEM; } - zPath = (const char*)zFree; - nPath++; + /* Check that we're not inserting a duplicate entry -OR- updating an + ** entry with a path, thereby making it into a duplicate. */ + if( (pOld==0 || bUpdate) && rc==SQLITE_OK ){ + ZipfileEntry *p; + for(p=pTab->pFirstEntry; p; p=p->pNext){ + if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){ + switch( sqlite3_vtab_on_conflict(pTab->db) ){ + case SQLITE_IGNORE: { + goto zipfile_update_done; + } + case SQLITE_REPLACE: { + pOld2 = p; + break; + } + default: { + zipfileTableErr(pTab, "duplicate name: \"%s\"", zPath); + rc = SQLITE_CONSTRAINT; + break; + } + } + break; + } + } } - } - /* Check that we're not inserting a duplicate entry */ - if( rc==SQLITE_OK ){ - ZipfileEntry *p; - for(p=pTab->pFirstEntry; p; p=p->pNext){ - if( p->bDeleted ) continue; - if( zipfileComparePath(p->zPath, zPath, nPath)==0 ){ - rc = SQLITE_CONSTRAINT; - break; + if( rc==SQLITE_OK ){ + /* Create the new CDS record. */ + pNew = zipfileNewEntry(zPath); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS; + pNew->cds.iCompression = (u16)iMethod; + zipfileMtimeToDos(&pNew->cds, mTime); + pNew->cds.crc32 = iCrc32; + pNew->cds.szCompressed = nData; + pNew->cds.szUncompressed = (u32)sz; + pNew->cds.iExternalAttr = (mode<<16); + pNew->cds.iOffset = (u32)pTab->szCurrent; + pNew->cds.nFile = (u16)nPath; + pNew->mUnixTime = (u32)mTime; + rc = zipfileAppendEntry(pTab, pNew, pData, nData); + zipfileAddEntry(pTab, pOld, pNew); } } } - if( rc==SQLITE_OK ){ - /* Create the new CDS record. */ - memset(&cds, 0, sizeof(cds)); - cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; - cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; - cds.flags = ZIPFILE_NEWENTRY_FLAGS; - cds.iCompression = (u16)iMethod; - zipfileMtimeToDos(&cds, (u32)mTime); - cds.crc32 = iCrc32; - cds.szCompressed = nData; - cds.szUncompressed = (u32)sz; - cds.iExternalAttr = (mode<<16); - cds.iOffset = (u32)pTab->szCurrent; - pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime); - if( pNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - zipfileAddEntry(pTab, pOld, pNew); + if( rc==SQLITE_OK && (pOld || pOld2) ){ + ZipfileCsr *pCsr; + for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ + if( pCsr->pCurrent && (pCsr->pCurrent==pOld || pCsr->pCurrent==pOld2) ){ + pCsr->pCurrent = pCsr->pCurrent->pNext; + pCsr->bNoop = 1; + } } - } - /* Append the new header+file to the archive */ - if( rc==SQLITE_OK ){ - rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime); + zipfileRemoveEntryFromList(pTab, pOld); + zipfileRemoveEntryFromList(pTab, pOld2); } - if( rc!=SQLITE_OK && pOld ){ - pOld->bDeleted = 0; - } +zipfile_update_done: sqlite3_free(pFree); sqlite3_free(zFree); return rc; } -static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ - u8 *aBuf = pTab->aBuffer; - - zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_EOCD); - zipfileWrite16(aBuf, p->iDisk); - zipfileWrite16(aBuf, p->iFirstDisk); - zipfileWrite16(aBuf, p->nEntry); - zipfileWrite16(aBuf, p->nEntryTotal); - zipfileWrite32(aBuf, p->nSize); - zipfileWrite32(aBuf, p->iOffset); - zipfileWrite16(aBuf, 0); /* Size of trailing comment in bytes*/ - - assert( (aBuf-pTab->aBuffer)==22 ); - return zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer)); -} - -static void zipfileCleanupTransaction(ZipfileTab *pTab){ - ZipfileEntry *pEntry; - ZipfileEntry *pNext; +static int zipfileSerializeEOCD(ZipfileEOCD *p, u8 *aBuf){ + u8 *a = aBuf; + zipfileWrite32(a, ZIPFILE_SIGNATURE_EOCD); + zipfileWrite16(a, p->iDisk); + zipfileWrite16(a, p->iFirstDisk); + zipfileWrite16(a, p->nEntry); + zipfileWrite16(a, p->nEntryTotal); + zipfileWrite32(a, p->nSize); + zipfileWrite32(a, p->iOffset); + zipfileWrite16(a, 0); /* Size of trailing comment in bytes*/ - for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){ - pNext = pEntry->pNext; - sqlite3_free(pEntry); - } - pTab->pFirstEntry = 0; - pTab->pLastEntry = 0; - fclose(pTab->pWriteFd); - pTab->pWriteFd = 0; - pTab->szCurrent = 0; - pTab->szOrig = 0; + return a-aBuf; } -static int zipfileBegin(sqlite3_vtab *pVtab){ - ZipfileTab *pTab = (ZipfileTab*)pVtab; - int rc = SQLITE_OK; - - assert( pTab->pWriteFd==0 ); - - /* This table is only writable if a default archive path was specified - ** as part of the CREATE VIRTUAL TABLE statement. */ - if( pTab->zFile==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: writing requires a default archive" - ); - return SQLITE_ERROR; - } - - /* Open a write fd on the file. Also load the entire central directory - ** structure into memory. During the transaction any new file data is - ** appended to the archive file, but the central directory is accumulated - ** in main-memory until the transaction is committed. */ - pTab->pWriteFd = fopen(pTab->zFile, "ab+"); - if( pTab->pWriteFd==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: failed to open file %s for writing", pTab->zFile - ); - rc = SQLITE_ERROR; +static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ + int nBuf = zipfileSerializeEOCD(p, pTab->aBuffer); + assert( nBuf==ZIPFILE_EOCD_FIXED_SZ ); + return zipfileAppendData(pTab, pTab->aBuffer, nBuf); +} + +/* +** Serialize the CDS structure into buffer aBuf[]. Return the number +** of bytes written. +*/ +static int zipfileSerializeCDS(ZipfileEntry *pEntry, u8 *aBuf){ + u8 *a = aBuf; + ZipfileCDS *pCDS = &pEntry->cds; + + if( pEntry->aExtra==0 ){ + pCDS->nExtra = 9; + } + + zipfileWrite32(a, ZIPFILE_SIGNATURE_CDS); + zipfileWrite16(a, pCDS->iVersionMadeBy); + zipfileWrite16(a, pCDS->iVersionExtract); + zipfileWrite16(a, pCDS->flags); + zipfileWrite16(a, pCDS->iCompression); + zipfileWrite16(a, pCDS->mTime); + zipfileWrite16(a, pCDS->mDate); + zipfileWrite32(a, pCDS->crc32); + zipfileWrite32(a, pCDS->szCompressed); + zipfileWrite32(a, pCDS->szUncompressed); + assert( a==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); + zipfileWrite16(a, pCDS->nFile); + zipfileWrite16(a, pCDS->nExtra); + zipfileWrite16(a, pCDS->nComment); + zipfileWrite16(a, pCDS->iDiskStart); + zipfileWrite16(a, pCDS->iInternalAttr); + zipfileWrite32(a, pCDS->iExternalAttr); + zipfileWrite32(a, pCDS->iOffset); + + memcpy(a, pCDS->zFile, pCDS->nFile); + a += pCDS->nFile; + + if( pEntry->aExtra ){ + int n = (int)pCDS->nExtra + (int)pCDS->nComment; + memcpy(a, pEntry->aExtra, n); + a += n; }else{ - fseek(pTab->pWriteFd, 0, SEEK_END); - pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); - rc = zipfileLoadDirectory(pTab); + assert( pCDS->nExtra==9 ); + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); } - if( rc!=SQLITE_OK ){ - zipfileCleanupTransaction(pTab); - } - - return rc; + return a-aBuf; } static int zipfileCommit(sqlite3_vtab *pVtab){ @@ -5483,10 +5990,10 @@ static int zipfileCommit(sqlite3_vtab *pVtab){ ZipfileEOCD eocd; int nEntry = 0; - /* Write out all undeleted entries */ + /* Write out all entries */ for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){ - if( p->bDeleted ) continue; - rc = zipfileAppendData(pTab, p->aCdsEntry, p->nCdsEntry); + int n = zipfileSerializeCDS(p, pTab->aBuffer); + rc = zipfileAppendData(pTab, pTab->aBuffer, n); nEntry++; } @@ -5527,7 +6034,7 @@ static void zipfileFunctionCds( pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0])); if( pCsr ){ - ZipfileCDS *p = &pCsr->cds; + ZipfileCDS *p = &pCsr->pCurrent->cds; char *zRes = sqlite3_mprintf("{" "\"version-made-by\" : %u, " "\"version-to-extract\" : %u, " @@ -5564,7 +6071,6 @@ static void zipfileFunctionCds( } } - /* ** xFindFunction method. */ @@ -5575,17 +6081,258 @@ static int zipfileFindFunction( void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ void **ppArg /* OUT: User data for *pxFunc */ ){ - if( nArg>0 ){ - if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ - *pxFunc = zipfileFunctionCds; - *ppArg = (void*)pVtab; - return 1; + if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ + *pxFunc = zipfileFunctionCds; + *ppArg = (void*)pVtab; + return 1; + } + return 0; +} + +typedef struct ZipfileBuffer ZipfileBuffer; +struct ZipfileBuffer { + u8 *a; /* Pointer to buffer */ + int n; /* Size of buffer in bytes */ + int nAlloc; /* Byte allocated at a[] */ +}; + +typedef struct ZipfileCtx ZipfileCtx; +struct ZipfileCtx { + int nEntry; + ZipfileBuffer body; + ZipfileBuffer cds; +}; + +static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){ + if( pBuf->n+nByte>pBuf->nAlloc ){ + u8 *aNew; + sqlite3_int64 nNew = pBuf->n ? pBuf->n*2 : 512; + int nReq = pBuf->n + nByte; + + while( nNew<nReq ) nNew = nNew*2; + aNew = sqlite3_realloc64(pBuf->a, nNew); + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->a = aNew; + pBuf->nAlloc = (int)nNew; + } + return SQLITE_OK; +} + +/* +** xStep() callback for the zipfile() aggregate. This can be called in +** any of the following ways: +** +** SELECT zipfile(name,data) ... +** SELECT zipfile(name,mode,mtime,data) ... +** SELECT zipfile(name,mode,mtime,data,method) ... +*/ +void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ + ZipfileCtx *p; /* Aggregate function context */ + ZipfileEntry e; /* New entry to add to zip archive */ + + sqlite3_value *pName = 0; + sqlite3_value *pMode = 0; + sqlite3_value *pMtime = 0; + sqlite3_value *pData = 0; + sqlite3_value *pMethod = 0; + + int bIsDir = 0; + u32 mode; + int rc = SQLITE_OK; + char *zErr = 0; + + int iMethod = -1; /* Compression method to use (0 or 8) */ + + const u8 *aData = 0; /* Possibly compressed data for new entry */ + int nData = 0; /* Size of aData[] in bytes */ + int szUncompressed = 0; /* Size of data before compression */ + u8 *aFree = 0; /* Free this before returning */ + u32 iCrc32 = 0; /* crc32 of uncompressed data */ + + char *zName = 0; /* Path (name) of new entry */ + int nName = 0; /* Size of zName in bytes */ + char *zFree = 0; /* Free this before returning */ + int nByte; + + memset(&e, 0, sizeof(e)); + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 ) return; + + /* Martial the arguments into stack variables */ + if( nVal!=2 && nVal!=4 && nVal!=5 ){ + zErr = sqlite3_mprintf("wrong number of arguments to function zipfile()"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + pName = apVal[0]; + if( nVal==2 ){ + pData = apVal[1]; + }else{ + pMode = apVal[1]; + pMtime = apVal[2]; + pData = apVal[3]; + if( nVal==5 ){ + pMethod = apVal[4]; } } - return 0; + /* Check that the 'name' parameter looks ok. */ + zName = (char*)sqlite3_value_text(pName); + nName = sqlite3_value_bytes(pName); + if( zName==0 ){ + zErr = sqlite3_mprintf("first argument to zipfile() must be non-NULL"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + + /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use + ** deflate compression) or NULL (choose automatically). */ + if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){ + iMethod = (int)sqlite3_value_int64(pMethod); + if( iMethod!=0 && iMethod!=8 ){ + zErr = sqlite3_mprintf("illegal method value: %d", iMethod); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + } + + /* Now inspect the data. If this is NULL, then the new entry must be a + ** directory. Otherwise, figure out whether or not the data should + ** be deflated or simply stored in the zip archive. */ + if( sqlite3_value_type(pData)==SQLITE_NULL ){ + bIsDir = 1; + iMethod = 0; + }else{ + aData = sqlite3_value_blob(pData); + szUncompressed = nData = sqlite3_value_bytes(pData); + iCrc32 = crc32(0, aData, nData); + if( iMethod<0 || iMethod==8 ){ + int nOut = 0; + rc = zipfileDeflate(aData, nData, &aFree, &nOut, &zErr); + if( rc!=SQLITE_OK ){ + goto zipfile_step_out; + } + if( iMethod==8 || nOut<nData ){ + aData = aFree; + nData = nOut; + iMethod = 8; + }else{ + iMethod = 0; + } + } + } + + /* Decode the "mode" argument. */ + rc = zipfileGetMode(pMode, bIsDir, &mode, &zErr); + if( rc ) goto zipfile_step_out; + + /* Decode the "mtime" argument. */ + e.mUnixTime = zipfileGetTime(pMtime); + + /* If this is a directory entry, ensure that there is exactly one '/' + ** at the end of the path. Or, if this is not a directory and the path + ** ends in '/' it is an error. */ + if( bIsDir==0 ){ + if( zName[nName-1]=='/' ){ + zErr = sqlite3_mprintf("non-directory name must not end with /"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + }else{ + if( zName[nName-1]!='/' ){ + zName = zFree = sqlite3_mprintf("%s/", zName); + nName++; + if( zName==0 ){ + rc = SQLITE_NOMEM; + goto zipfile_step_out; + } + }else{ + while( nName>1 && zName[nName-2]=='/' ) nName--; + } + } + + /* Assemble the ZipfileEntry object for the new zip archive entry */ + e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + e.cds.flags = ZIPFILE_NEWENTRY_FLAGS; + e.cds.iCompression = (u16)iMethod; + zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime); + e.cds.crc32 = iCrc32; + e.cds.szCompressed = nData; + e.cds.szUncompressed = szUncompressed; + e.cds.iExternalAttr = (mode<<16); + e.cds.iOffset = p->body.n; + e.cds.nFile = (u16)nName; + e.cds.zFile = zName; + + /* Append the LFH to the body of the new archive */ + nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out; + p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]); + + /* Append the data to the body of the new archive */ + if( nData>0 ){ + if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out; + memcpy(&p->body.a[p->body.n], aData, nData); + p->body.n += nData; + } + + /* Append the CDS record to the directory of the new archive */ + nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out; + p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]); + + /* Increment the count of entries in the archive */ + p->nEntry++; + + zipfile_step_out: + sqlite3_free(aFree); + sqlite3_free(zFree); + if( rc ){ + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + } + sqlite3_free(zErr); +} + +/* +** xFinalize() callback for zipfile aggregate function. +*/ +void zipfileFinal(sqlite3_context *pCtx){ + ZipfileCtx *p; + ZipfileEOCD eocd; + sqlite3_int64 nZip; + u8 *aZip; + + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 ) return; + if( p->nEntry>0 ){ + memset(&eocd, 0, sizeof(eocd)); + eocd.nEntry = (u16)p->nEntry; + eocd.nEntryTotal = (u16)p->nEntry; + eocd.nSize = p->cds.n; + eocd.iOffset = p->body.n; + + nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ; + aZip = (u8*)sqlite3_malloc64(nZip); + if( aZip==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + memcpy(aZip, p->body.a, p->body.n); + memcpy(&aZip[p->body.n], p->cds.a, p->cds.n); + zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]); + sqlite3_result_blob(pCtx, aZip, (int)nZip, zipfileFree); + } + } + + sqlite3_free(p->body.a); + sqlite3_free(p->cds.a); } + /* ** Register the "zipfile" virtual table. */ @@ -5603,7 +6350,7 @@ static int zipfileRegister(sqlite3 *db){ zipfileNext, /* xNext - advance a cursor */ zipfileEof, /* xEof - check for end of scan */ zipfileColumn, /* xColumn - read data */ - zipfileRowid, /* xRowid - read data */ + 0, /* xRowid - read data */ zipfileUpdate, /* xUpdate */ zipfileBegin, /* xBegin */ 0, /* xSync */ @@ -5614,8 +6361,11 @@ static int zipfileRegister(sqlite3 *db){ }; int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); + if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); if( rc==SQLITE_OK ){ - rc = sqlite3_overload_function(db, "zipfile_cds", -1); + rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, + zipfileStep, zipfileFinal + ); } return rc; } @@ -6577,6 +7327,7 @@ static int idxRegisterVtab(sqlite3expert *p){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ + 0, /* xShadowName */ }; return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p); @@ -7056,9 +7807,9 @@ int idxFindIndexes( "EXPLAIN QUERY PLAN %s", pStmt->zSql ); while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){ - int iSelectid = sqlite3_column_int(pExplain, 0); - int iOrder = sqlite3_column_int(pExplain, 1); - int iFrom = sqlite3_column_int(pExplain, 2); + /* int iId = sqlite3_column_int(pExplain, 0); */ + /* int iParent = sqlite3_column_int(pExplain, 1); */ + /* int iNotUsed = sqlite3_column_int(pExplain, 2); */ const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3); int nDetail = STRLEN(zDetail); int i; @@ -7085,9 +7836,9 @@ int idxFindIndexes( } } - pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%d|%d|%d|%s\n", - iSelectid, iOrder, iFrom, zDetail - ); + if( zDetail[0]!='-' ){ + pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%s\n", zDetail); + } } for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){ @@ -7917,6 +8668,23 @@ struct ExpertInfo { int bVerbose; }; +/* A single line in the EQP output */ +typedef struct EQPGraphRow EQPGraphRow; +struct EQPGraphRow { + int iEqpId; /* ID for this row */ + int iParentId; /* ID of the parent row */ + EQPGraphRow *pNext; /* Next row in sequence */ + char zText[1]; /* Text to display for this row */ +}; + +/* All EQP output is collected into an instance of the following */ +typedef struct EQPGraph EQPGraph; +struct EQPGraph { + EQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ + EQPGraphRow *pLast; /* Last element of the pRow list */ + char zPrefix[100]; /* Graph prefix */ +}; + /* ** State information about the database connection is contained in an ** instance of the following structure. @@ -7926,12 +8694,19 @@ struct ShellState { sqlite3 *db; /* The database */ u8 autoExplain; /* Automatically turn on .explain mode */ u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ + u8 autoEQPtest; /* autoEQP is in test mode */ + u8 autoEQPtrace; /* autoEQP is in trace mode */ u8 statsOn; /* True to display memory stats before each finalize */ u8 scanstatsOn; /* True to display scan stats before each finalize */ u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ + u8 nEqpLevel; /* Depth of the EQP output graph */ + u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ + unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ + int lineno; /* Line number of last line read from in */ + FILE *in; /* Read commands from this stream */ FILE *out; /* Write results here */ FILE *traceOut; /* Output for sqlite3_trace() */ int nErr; /* Number of errors seen */ @@ -7942,7 +8717,11 @@ struct ShellState { int writableSchema; /* True if PRAGMA writable_schema=ON */ int showHeader; /* True to show column names in List or Column mode */ int nCheck; /* Number of ".check" commands run */ + unsigned nProgress; /* Number of progress callbacks encountered */ + unsigned mxProgress; /* Maximum progress callbacks before failing */ + unsigned flgProgress; /* Flags for the progress callback */ unsigned shellFlgs; /* Various flags */ + sqlite3_int64 szMax; /* --maxsize argument to .open */ char *zDestTable; /* Name of destination table when MODE_Insert */ char *zTempFile; /* Temporary file that might need deleting */ char zTestcase[30]; /* Name of current test case */ @@ -7963,6 +8742,7 @@ struct ShellState { int *aiIndent; /* Array of indents used in MODE_Explain */ int nIndent; /* Size of array aiIndent[] */ int iIndent; /* Index of current op in aiIndent[] */ + EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ #if defined(SQLITE_ENABLE_SESSION) int nSession; /* Number of active sessions */ OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ @@ -7973,17 +8753,33 @@ struct ShellState { /* Allowed values for ShellState.autoEQP */ -#define AUTOEQP_off 0 -#define AUTOEQP_on 1 -#define AUTOEQP_trigger 2 -#define AUTOEQP_full 3 +#define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ +#define AUTOEQP_on 1 /* Automatic EQP is on */ +#define AUTOEQP_trigger 2 /* On and also show plans for triggers */ +#define AUTOEQP_full 3 /* Show full EXPLAIN */ /* Allowed values for ShellState.openMode */ -#define SHELL_OPEN_UNSPEC 0 /* No open-mode specified */ -#define SHELL_OPEN_NORMAL 1 /* Normal database file */ -#define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ -#define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ +#define SHELL_OPEN_UNSPEC 0 /* No open-mode specified */ +#define SHELL_OPEN_NORMAL 1 /* Normal database file */ +#define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ +#define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ +#define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ +#define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ +#define SHELL_OPEN_HEXDB 6 /* Use "dbtotxt" output as data source */ + +/* Allowed values for ShellState.eTraceType +*/ +#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */ +#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */ +#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */ + +/* Bits in the ShellState.flgProgress variable */ +#define SHELL_PROGRESS_QUIET 0x01 /* Omit announcing every progress callback */ +#define SHELL_PROGRESS_RESET 0x02 /* Reset the count when the progres + ** callback limit is reached, and for each + ** top-level SQL statement */ +#define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ /* ** These are the allowed shellFlgs values @@ -8018,6 +8814,7 @@ struct ShellState { #define MODE_Explain 9 /* Like MODE_Column, but do not truncate data */ #define MODE_Ascii 10 /* Use ASCII unit and record separators (0x1F/0x1E) */ #define MODE_Pretty 11 /* Pretty-print schemas */ +#define MODE_EQP 12 /* Converts EXPLAIN QUERY PLAN output into a graph */ static const char *modeDescr[] = { "line", @@ -8032,6 +8829,7 @@ static const char *modeDescr[] = { "explain", "ascii", "prettyprint", + "eqp" }; /* @@ -8090,6 +8888,7 @@ static void shellPutsFunc( ** ** Also throw an error if the EDITOR program returns a non-zero exit code. */ +#ifndef SQLITE_NOHAVE_SYSTEM static void editFunc( sqlite3_context *context, int argc, @@ -8101,6 +8900,7 @@ static void editFunc( char *zCmd = 0; int bBin; int rc; + int hasCRNL = 0; FILE *f = 0; sqlite3_int64 sz; sqlite3_int64 x; @@ -8132,6 +8932,8 @@ static void editFunc( } } bBin = sqlite3_value_type(argv[0])==SQLITE_BLOB; + /* When writing the file to be edited, do \n to \r\n conversions on systems + ** that want \r\n line endings */ f = fopen(zTempFile, bBin ? "wb" : "w"); if( f==0 ){ sqlite3_result_error(context, "edit() cannot open temp file", -1); @@ -8141,6 +8943,9 @@ static void editFunc( if( bBin ){ x = fwrite(sqlite3_value_blob(argv[0]), 1, sz, f); }else{ + const char *z = (const char*)sqlite3_value_text(argv[0]); + /* Remember whether or not the value originally contained \r\n */ + if( z && strstr(z,"\r\n")!=0 ) hasCRNL = 1; x = fwrite(sqlite3_value_text(argv[0]), 1, sz, f); } fclose(f); @@ -8160,7 +8965,7 @@ static void editFunc( sqlite3_result_error(context, "EDITOR returned non-zero", -1); goto edit_func_end; } - f = fopen(zTempFile, bBin ? "rb" : "r"); + f = fopen(zTempFile, "rb"); if( f==0 ){ sqlite3_result_error(context, "edit() cannot reopen temp file after edit", -1); @@ -8174,12 +8979,7 @@ static void editFunc( sqlite3_result_error_nomem(context); goto edit_func_end; } - if( bBin ){ - x = fread(p, 1, sz, f); - }else{ - x = fread(p, 1, sz, f); - p[sz] = 0; - } + x = fread(p, 1, sz, f); fclose(f); f = 0; if( x!=sz ){ @@ -8187,9 +8987,24 @@ static void editFunc( goto edit_func_end; } if( bBin ){ - sqlite3_result_blob(context, p, sz, sqlite3_free); + sqlite3_result_blob64(context, p, sz, sqlite3_free); }else{ - sqlite3_result_text(context, (const char*)p, sz, sqlite3_free); + sqlite3_int64 i, j; + if( hasCRNL ){ + /* If the original contains \r\n then do no conversions back to \n */ + j = sz; + }else{ + /* If the file did not originally contain \r\n then convert any new + ** \r\n back into \n */ + for(i=j=0; i<sz; i++){ + if( p[i]=='\r' && p[i+1]=='\n' ) i++; + p[j++] = p[i]; + } + sz = j; + p[sz] = 0; + } + sqlite3_result_text64(context, (const char*)p, sz, + sqlite3_free, SQLITE_UTF8); } p = 0; @@ -8199,6 +9014,7 @@ edit_func_end: sqlite3_free(zTempFile); sqlite3_free(p); } +#endif /* SQLITE_NOHAVE_SYSTEM */ /* ** Save or restore the current output mode @@ -8577,7 +9393,113 @@ static int wsToEol(const char *z){ } return 1; } - + +/* +** Add a new entry to the EXPLAIN QUERY PLAN data +*/ +static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){ + EQPGraphRow *pNew; + int nText = strlen30(zText); + if( p->autoEQPtest ){ + utf8_printf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); + } + pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); + if( pNew==0 ) shell_out_of_memory(); + pNew->iEqpId = iEqpId; + pNew->iParentId = p2; + memcpy(pNew->zText, zText, nText+1); + pNew->pNext = 0; + if( p->sGraph.pLast ){ + p->sGraph.pLast->pNext = pNew; + }else{ + p->sGraph.pRow = pNew; + } + p->sGraph.pLast = pNew; +} + +/* +** Free and reset the EXPLAIN QUERY PLAN data that has been collected +** in p->sGraph. +*/ +static void eqp_reset(ShellState *p){ + EQPGraphRow *pRow, *pNext; + for(pRow = p->sGraph.pRow; pRow; pRow = pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + memset(&p->sGraph, 0, sizeof(p->sGraph)); +} + +/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after +** pOld, or return the first such line if pOld is NULL +*/ +static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ + EQPGraphRow *pRow = pOld ? pOld->pNext : p->sGraph.pRow; + while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; + return pRow; +} + +/* Render a single level of the graph that has iEqpId as its parent. Called +** recursively to render sublevels. +*/ +static void eqp_render_level(ShellState *p, int iEqpId){ + EQPGraphRow *pRow, *pNext; + int n = strlen30(p->sGraph.zPrefix); + char *z; + for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ + pNext = eqp_next_row(p, iEqpId, pRow); + z = pRow->zText; + utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z); + if( n<(int)sizeof(p->sGraph.zPrefix)-7 ){ + memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); + eqp_render_level(p, pRow->iEqpId); + p->sGraph.zPrefix[n] = 0; + } + } +} + +/* +** Display and reset the EXPLAIN QUERY PLAN data +*/ +static void eqp_render(ShellState *p){ + EQPGraphRow *pRow = p->sGraph.pRow; + if( pRow ){ + if( pRow->zText[0]=='-' ){ + if( pRow->pNext==0 ){ + eqp_reset(p); + return; + } + utf8_printf(p->out, "%s\n", pRow->zText+3); + p->sGraph.pRow = pRow->pNext; + sqlite3_free(pRow); + }else{ + utf8_printf(p->out, "QUERY PLAN\n"); + } + p->sGraph.zPrefix[0] = 0; + eqp_render_level(p, 0); + eqp_reset(p); + } +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** Progress handler callback. +*/ +static int progress_handler(void *pClientData) { + ShellState *p = (ShellState*)pClientData; + p->nProgress++; + if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ + raw_printf(p->out, "Progress limit reached (%u)\n", p->nProgress); + if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; + if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; + return 1; + } + if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ + raw_printf(p->out, "Progress %u\n", p->nProgress); + } + return 0; +} +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ /* ** This is the callback routine that the shell @@ -8859,8 +9781,16 @@ static int shell_callback( }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); - sqlite3_snprintf(50,z,"%!.20g", r); - raw_printf(p->out, "%s", z); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + raw_printf(p->out, "1e999"); + }else if( ur==0xfff0000000000000LL ){ + raw_printf(p->out, "-1e999"); + }else{ + sqlite3_snprintf(50,z,"%!.20g", r); + raw_printf(p->out, "%s", z); + } }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); @@ -8928,6 +9858,10 @@ static int shell_callback( utf8_printf(p->out, "%s", p->rowSeparator); break; } + case MODE_EQP: { + eqp_append(p, atoi(azArg[0]), atoi(azArg[1]), azArg[3]); + break; + } } return 0; } @@ -9026,10 +9960,7 @@ static void set_table_name(ShellState *p, const char *zName){ n = strlen30(zName); if( cQuote ) n += n+2; z = p->zDestTable = malloc( n+1 ); - if( z==0 ){ - raw_printf(stderr,"Error: out of memory\n"); - exit(1); - } + if( z==0 ) shell_out_of_memory(); n = 0; if( cQuote ) z[n++] = cQuote; for(i=0; zName[i]; i++){ @@ -9184,29 +10115,54 @@ static int display_stats( ){ int iCur; int iHiwtr; + FILE *out; + if( pArg==0 || pArg->out==0 ) return 0; + out = pArg->out; + + if( pArg->pStmt && (pArg->statsOn & 2) ){ + int nCol, i, x; + sqlite3_stmt *pStmt = pArg->pStmt; + char z[100]; + nCol = sqlite3_column_count(pStmt); + raw_printf(out, "%-36s %d\n", "Number of output columns:", nCol); + for(i=0; i<nCol; i++){ + sqlite3_snprintf(sizeof(z),z,"Column %d %nname:", i, &x); + utf8_printf(out, "%-36s %s\n", z, sqlite3_column_name(pStmt,i)); +#ifndef SQLITE_OMIT_DECLTYPE + sqlite3_snprintf(30, z+x, "declared type:"); + utf8_printf(out, "%-36s %s\n", z, sqlite3_column_decltype(pStmt, i)); +#endif +#ifdef SQLITE_ENABLE_COLUMN_METADATA + sqlite3_snprintf(30, z+x, "database name:"); + utf8_printf(out, "%-36s %s\n", z, sqlite3_column_database_name(pStmt,i)); + sqlite3_snprintf(30, z+x, "table name:"); + utf8_printf(out, "%-36s %s\n", z, sqlite3_column_table_name(pStmt,i)); + sqlite3_snprintf(30, z+x, "origin name:"); + utf8_printf(out, "%-36s %s\n", z, sqlite3_column_origin_name(pStmt,i)); +#endif + } + } - if( pArg && pArg->out ){ - displayStatLine(pArg, "Memory Used:", - "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); - displayStatLine(pArg, "Number of Outstanding Allocations:", - "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); - if( pArg->shellFlgs & SHFLG_Pagecache ){ - displayStatLine(pArg, "Number of Pcache Pages Used:", - "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); - } - displayStatLine(pArg, "Number of Pcache Overflow Bytes:", - "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); - displayStatLine(pArg, "Largest Allocation:", - "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); - displayStatLine(pArg, "Largest Pcache Allocation:", - "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); + displayStatLine(pArg, "Memory Used:", + "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); + displayStatLine(pArg, "Number of Outstanding Allocations:", + "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); + if( pArg->shellFlgs & SHFLG_Pagecache ){ + displayStatLine(pArg, "Number of Pcache Pages Used:", + "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); + } + displayStatLine(pArg, "Number of Pcache Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); + displayStatLine(pArg, "Largest Allocation:", + "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); + displayStatLine(pArg, "Largest Pcache Allocation:", + "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH - displayStatLine(pArg, "Deepest Parser Stack:", - "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); + displayStatLine(pArg, "Deepest Parser Stack:", + "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif - } - if( pArg && pArg->out && db ){ + if( db ){ if( pArg->shellFlgs & SHFLG_Lookaside ){ iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, @@ -9241,6 +10197,9 @@ static int display_stats( sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); raw_printf(pArg->out, "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); + raw_printf(pArg->out, "Page cache spills: %d\n", iCur); + iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); raw_printf(pArg->out, "Schema Heap Usage: %d bytes\n", iCur); @@ -9250,7 +10209,7 @@ static int display_stats( iCur); } - if( pArg && pArg->out && db && pArg->pStmt ){ + if( pArg->pStmt ){ iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); raw_printf(pArg->out, "Fullscan Steps: %d\n", iCur); @@ -9260,6 +10219,12 @@ static int display_stats( raw_printf(pArg->out, "Autoindex Inserts: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); raw_printf(pArg->out, "Virtual Machine Steps: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE, bReset); + raw_printf(pArg->out, "Reprepare operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); + raw_printf(pArg->out, "Number of times run: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); + raw_printf(pArg->out, "Memory used by prepared stmt: %d\n", iCur); } #ifdef __linux__ @@ -9358,8 +10323,7 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ int iOp; /* Index of operation in p->aiIndent[] */ - const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", - "NextIfOpen", "PrevIfOpen", 0 }; + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 }; const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; @@ -9409,7 +10373,9 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ } nAlloc += 100; p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); + if( p->aiIndent==0 ) shell_out_of_memory(); abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); + if( abYield==0 ) shell_out_of_memory(); } abYield[iOp] = str_in_array(zOp, azYield); p->aiIndent[iOp] = 0; @@ -9470,13 +10436,72 @@ static void restore_debug_trace_modes(void){ #endif } +/* Create the TEMP table used to store parameter bindings */ +static void bind_table_init(ShellState *p){ + int wrSchema = 0; + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, -1, &wrSchema); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, 0); + sqlite3_exec(p->db, + "CREATE TABLE IF NOT EXISTS temp.sqlite_parameters(\n" + " key TEXT PRIMARY KEY,\n" + " value ANY\n" + ") WITHOUT ROWID;", + 0, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, wrSchema, 0); +} + +/* +** Bind parameters on a prepared statement. +** +** Parameter bindings are taken from a TEMP table of the form: +** +** CREATE TEMP TABLE sqlite_parameters(key TEXT PRIMARY KEY, value) +** WITHOUT ROWID; +** +** No bindings occur if this table does not exist. The special character '$' +** is included in the table name to help prevent collisions with actual tables. +** The table must be in the TEMP schema. +*/ +static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ + int nVar; + int i; + int rc; + sqlite3_stmt *pQ = 0; + + nVar = sqlite3_bind_parameter_count(pStmt); + if( nVar==0 ) return; /* Nothing to do */ + if( sqlite3_table_column_metadata(pArg->db, "TEMP", "sqlite_parameters", + "key", 0, 0, 0, 0, 0)!=SQLITE_OK ){ + return; /* Parameter table does not exist */ + } + rc = sqlite3_prepare_v2(pArg->db, + "SELECT value FROM temp.sqlite_parameters" + " WHERE key=?1", -1, &pQ, 0); + if( rc || pQ==0 ) return; + for(i=1; i<=nVar; i++){ + char zNum[30]; + const char *zVar = sqlite3_bind_parameter_name(pStmt, i); + if( zVar==0 ){ + sqlite3_snprintf(sizeof(zNum),zNum,"?%d",i); + zVar = zNum; + } + sqlite3_bind_text(pQ, 1, zVar, -1, SQLITE_STATIC); + if( sqlite3_step(pQ)==SQLITE_ROW ){ + sqlite3_bind_value(pStmt, i, sqlite3_column_value(pQ, 0)); + }else{ + sqlite3_bind_null(pStmt, i); + } + sqlite3_reset(pQ); + } + sqlite3_finalize(pQ); +} + /* ** Run a prepared statement */ static void exec_prepared_stmt( ShellState *pArg, /* Pointer to ShellState */ - sqlite3_stmt *pStmt, /* Statment to run */ - int (*xCallback)(void*,int,char**,char**,int*) /* Callback function */ + sqlite3_stmt *pStmt /* Statment to run */ ){ int rc; @@ -9486,54 +10511,47 @@ static void exec_prepared_stmt( rc = sqlite3_step(pStmt); /* if we have a result set... */ if( SQLITE_ROW == rc ){ - /* if we have a callback... */ - if( xCallback ){ - /* allocate space for col name ptr, value ptr, and type */ - int nCol = sqlite3_column_count(pStmt); - void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); - if( !pData ){ - rc = SQLITE_NOMEM; - }else{ - char **azCols = (char **)pData; /* Names of result columns */ - char **azVals = &azCols[nCol]; /* Results */ - int *aiTypes = (int *)&azVals[nCol]; /* Result types */ - int i, x; - assert(sizeof(int) <= sizeof(char *)); - /* save off ptrs to column names */ + /* allocate space for col name ptr, value ptr, and type */ + int nCol = sqlite3_column_count(pStmt); + void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); + if( !pData ){ + rc = SQLITE_NOMEM; + }else{ + char **azCols = (char **)pData; /* Names of result columns */ + char **azVals = &azCols[nCol]; /* Results */ + int *aiTypes = (int *)&azVals[nCol]; /* Result types */ + int i, x; + assert(sizeof(int) <= sizeof(char *)); + /* save off ptrs to column names */ + for(i=0; i<nCol; i++){ + azCols[i] = (char *)sqlite3_column_name(pStmt, i); + } + do{ + /* extract the data and data types */ for(i=0; i<nCol; i++){ - azCols[i] = (char *)sqlite3_column_name(pStmt, i); - } - do{ - /* extract the data and data types */ - for(i=0; i<nCol; i++){ - aiTypes[i] = x = sqlite3_column_type(pStmt, i); - if( x==SQLITE_BLOB && pArg && pArg->cMode==MODE_Insert ){ - azVals[i] = ""; - }else{ - azVals[i] = (char*)sqlite3_column_text(pStmt, i); - } - if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ - rc = SQLITE_NOMEM; - break; /* from for */ - } - } /* end for */ + aiTypes[i] = x = sqlite3_column_type(pStmt, i); + if( x==SQLITE_BLOB && pArg && pArg->cMode==MODE_Insert ){ + azVals[i] = ""; + }else{ + azVals[i] = (char*)sqlite3_column_text(pStmt, i); + } + if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ + rc = SQLITE_NOMEM; + break; /* from for */ + } + } /* end for */ - /* if data and types extracted successfully... */ - if( SQLITE_ROW == rc ){ - /* call the supplied callback with the result row data */ - if( xCallback(pArg, nCol, azVals, azCols, aiTypes) ){ - rc = SQLITE_ABORT; - }else{ - rc = sqlite3_step(pStmt); - } + /* if data and types extracted successfully... */ + if( SQLITE_ROW == rc ){ + /* call the supplied callback with the result row data */ + if( shell_callback(pArg, nCol, azVals, azCols, aiTypes) ){ + rc = SQLITE_ABORT; + }else{ + rc = sqlite3_step(pStmt); } - } while( SQLITE_ROW == rc ); - sqlite3_free(pData); - } - }else{ - do{ - rc = sqlite3_step(pStmt); - } while( rc == SQLITE_ROW ); + } + } while( SQLITE_ROW == rc ); + sqlite3_free(pData); } } } @@ -9679,17 +10697,15 @@ static int expertDotCommand( ** and callback data argument. */ static int shell_exec( - sqlite3 *db, /* An open database */ - const char *zSql, /* SQL to be evaluated */ - int (*xCallback)(void*,int,char**,char**,int*), /* Callback function */ - /* (not the same as sqlite3_exec) */ ShellState *pArg, /* Pointer to ShellState */ + const char *zSql, /* SQL to be evaluated */ char **pzErrMsg /* Error msg written here */ ){ sqlite3_stmt *pStmt = NULL; /* Statement to execute. */ int rc = SQLITE_OK; /* Return Code */ int rc2; const char *zLeftover; /* Tail of unprocessed SQL */ + sqlite3 *db = pArg->db; if( pzErrMsg ){ *pzErrMsg = NULL; @@ -9732,7 +10748,7 @@ static int shell_exec( } /* Show the EXPLAIN QUERY PLAN if .eqp is on */ - if( pArg && pArg->autoEQP && sqlite3_strlike("EXPLAIN%",zStmtSql,0)!=0 ){ + if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){ sqlite3_stmt *pExplain; char *zEQP; int triggerEQP = 0; @@ -9745,11 +10761,13 @@ static int shell_exec( rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); if( rc==SQLITE_OK ){ while( sqlite3_step(pExplain)==SQLITE_ROW ){ - raw_printf(pArg->out,"--EQP-- %d,",sqlite3_column_int(pExplain, 0)); - raw_printf(pArg->out,"%d,", sqlite3_column_int(pExplain, 1)); - raw_printf(pArg->out,"%d,", sqlite3_column_int(pExplain, 2)); - utf8_printf(pArg->out,"%s\n", sqlite3_column_text(pExplain, 3)); + const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); + int iEqpId = sqlite3_column_int(pExplain, 0); + int iParentId = sqlite3_column_int(pExplain, 1); + if( zEQPLine[0]=='-' ) eqp_render(pArg); + eqp_append(pArg, iEqpId, iParentId, zEQPLine); } + eqp_render(pArg); } sqlite3_finalize(pExplain); sqlite3_free(zEQP); @@ -9760,23 +10778,31 @@ static int shell_exec( if( rc==SQLITE_OK ){ pArg->cMode = MODE_Explain; explain_data_prepare(pArg, pExplain); - exec_prepared_stmt(pArg, pExplain, xCallback); + exec_prepared_stmt(pArg, pExplain); explain_data_delete(pArg); } sqlite3_finalize(pExplain); sqlite3_free(zEQP); } - sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, triggerEQP, 0); + if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ + sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0); + /* Reprepare pStmt before reactiving trace modes */ + sqlite3_finalize(pStmt); + sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( pArg ) pArg->pStmt = pStmt; + } restore_debug_trace_modes(); } if( pArg ){ pArg->cMode = pArg->mode; - if( pArg->autoExplain - && sqlite3_column_count(pStmt)==8 - && sqlite3_strlike("EXPLAIN%", zStmtSql,0)==0 - ){ - pArg->cMode = MODE_Explain; + if( pArg->autoExplain ){ + if( sqlite3_stmt_isexplain(pStmt)==1 ){ + pArg->cMode = MODE_Explain; + } + if( sqlite3_stmt_isexplain(pStmt)==2 ){ + pArg->cMode = MODE_EQP; + } } /* If the shell is currently in ".explain" mode, gather the extra @@ -9786,8 +10812,10 @@ static int shell_exec( } } - exec_prepared_stmt(pArg, pStmt, xCallback); + bind_prepared_stmt(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); explain_data_delete(pArg); + eqp_render(pArg); /* print usage stats if stats on */ if( pArg && pArg->statsOn ){ @@ -9865,10 +10893,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){ if( nCol>=nAlloc-2 ){ nAlloc = nAlloc*2 + nCol + 10; azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); - if( azCol==0 ){ - raw_printf(stderr, "Error: out of memory\n"); - exit(1); - } + if( azCol==0 ) shell_out_of_memory(); } azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); if( sqlite3_column_int(pStmt, 5) ){ @@ -10049,11 +11074,11 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ savedMode = p->mode; p->zDestTable = sTable.z; p->mode = p->cMode = MODE_Insert; - rc = shell_exec(p->db, sSelect.z, shell_callback, p, 0); + rc = shell_exec(p, sSelect.z, 0); if( (rc&0xff)==SQLITE_CORRUPT ){ raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); toggleSelectOrder(p->db); - shell_exec(p->db, sSelect.z, shell_callback, p, 0); + shell_exec(p, sSelect.z, 0); toggleSelectOrder(p->db); } p->zDestTable = savedDestTable; @@ -10104,133 +11129,280 @@ static int run_schema_dump_query( } /* -** Text of a help message +** Text of help messages. +** +** The help text for each individual command begins with a line that starts +** with ".". Subsequent lines are supplimental information. +** +** There must be two or more spaces between the end of the command and the +** start of the description of what that command does. */ -static char zHelp[] = +static const char *(azHelp[]) = { #if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) - ".archive ... Manage SQL archives: \".archive --help\" for details\n" + ".archive ... Manage SQL archives", + " Each command must have exactly one of the following options:", + " -c, --create Create a new archive", + " -u, --update Add files or update files with changed mtime", + " -i, --insert Like -u but always add even if mtime unchanged", + " -t, --list List contents of archive", + " -x, --extract Extract files from archive", + " Optional arguments:", + " -v, --verbose Print each filename as it is processed", + " -f FILE, --file FILE Operate on archive FILE (default is current db)", + " -a FILE, --append FILE Operate on FILE opened using the apndvfs VFS", + " -C DIR, --directory DIR Change to directory DIR to read/extract files", + " -n, --dryrun Show the SQL that would have occurred", + " Examples:", + " .ar -cf archive.sar foo bar # Create archive.sar from files foo and bar", + " .ar -tf archive.sar # List members of archive.sar", + " .ar -xvf archive.sar # Verbosely extract files from archive.sar", + " See also:", + " http://sqlite.org/cli.html#sqlar_archive_support", #endif #ifndef SQLITE_OMIT_AUTHORIZATION - ".auth ON|OFF Show authorizer callbacks\n" -#endif - ".backup ?DB? FILE Backup DB (default \"main\") to FILE\n" - ".bail on|off Stop after hitting an error. Default OFF\n" - ".binary on|off Turn binary output on or off. Default OFF\n" - ".cd DIRECTORY Change the working directory to DIRECTORY\n" - ".changes on|off Show number of rows changed by SQL\n" - ".check GLOB Fail if output since .testcase does not match\n" - ".clone NEWDB Clone data into NEWDB from the existing database\n" - ".databases List names and files of attached databases\n" - ".dbinfo ?DB? Show status information about the database\n" - ".dump ?TABLE? ... Dump the database in an SQL text format\n" - " If TABLE specified, only dump tables matching\n" - " LIKE pattern TABLE.\n" - ".echo on|off Turn command echo on or off\n" - ".eqp on|off|full Enable or disable automatic EXPLAIN QUERY PLAN\n" - ".excel Display the output of next command in a spreadsheet\n" - ".exit Exit this program\n" - ".expert EXPERIMENTAL. Suggest indexes for specified queries\n" + ".auth ON|OFF Show authorizer callbacks", +#endif + ".backup ?DB? FILE Backup DB (default \"main\") to FILE", + " --append Use the appendvfs", + " --async Write to FILE without a journal and without fsync()", + ".bail on|off Stop after hitting an error. Default OFF", + ".binary on|off Turn binary output on or off. Default OFF", + ".cd DIRECTORY Change the working directory to DIRECTORY", + ".changes on|off Show number of rows changed by SQL", + ".check GLOB Fail if output since .testcase does not match", + ".clone NEWDB Clone data into NEWDB from the existing database", + ".databases List names and files of attached databases", + ".dbconfig ?op? ?val? List or change sqlite3_db_config() options", + ".dbinfo ?DB? Show status information about the database", + ".dump ?TABLE? ... Render all database content as SQL", + " Options:", + " --preserve-rowids Include ROWID values in the output", + " --newlines Allow unescaped newline characters in output", + " TABLE is LIKE pattern for the tables to dump", + ".echo on|off Turn command echo on or off", + ".eqp on|off|full|... Enable or disable automatic EXPLAIN QUERY PLAN", + " Other Modes:", +#ifdef SQLITE_DEBUG + " test Show raw EXPLAIN QUERY PLAN output", + " trace Like \"full\" but also enable \"PRAGMA vdbe_trace\"", +#endif + " trigger Like \"full\" but also show trigger bytecode", + ".excel Display the output of next command in a spreadsheet", + ".exit ?CODE? Exit this program with return-code CODE", + ".expert EXPERIMENTAL. Suggest indexes for specified queries", /* Because explain mode comes on automatically now, the ".explain" mode ** is removed from the help screen. It is still supported for legacy, however */ -/*".explain ?on|off|auto? Turn EXPLAIN output mode on or off or to automatic\n"*/ - ".fullschema ?--indent? Show schema and the content of sqlite_stat tables\n" - ".headers on|off Turn display of headers on or off\n" - ".help Show this message\n" - ".import FILE TABLE Import data from FILE into TABLE\n" +/*".explain ?on|off|auto? Turn EXPLAIN output mode on or off or to automatic",*/ + ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", + ".headers on|off Turn display of headers on or off", + ".help ?-all? ?PATTERN? Show help text for PATTERN", + ".import FILE TABLE Import data from FILE into TABLE", #ifndef SQLITE_OMIT_TEST_CONTROL - ".imposter INDEX TABLE Create imposter table TABLE on index INDEX\n" + ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", #endif - ".indexes ?TABLE? Show names of all indexes\n" - " If TABLE specified, only show indexes for tables\n" - " matching LIKE pattern TABLE.\n" + ".indexes ?TABLE? Show names of indexes", + " If TABLE is specified, only show indexes for", + " tables matching TABLE using the LIKE operator.", #ifdef SQLITE_ENABLE_IOTRACE - ".iotrace FILE Enable I/O diagnostic logging to FILE\n" + ".iotrace FILE Enable I/O diagnostic logging to FILE", #endif - ".limit ?LIMIT? ?VAL? Display or change the value of an SQLITE_LIMIT\n" - ".lint OPTIONS Report potential schema issues. Options:\n" - " fkey-indexes Find missing foreign key indexes\n" + ".limit ?LIMIT? ?VAL? Display or change the value of an SQLITE_LIMIT", + ".lint OPTIONS Report potential schema issues.", + " Options:", + " fkey-indexes Find missing foreign key indexes", #ifndef SQLITE_OMIT_LOAD_EXTENSION - ".load FILE ?ENTRY? Load an extension library\n" -#endif - ".log FILE|off Turn logging on or off. FILE can be stderr/stdout\n" - ".mode MODE ?TABLE? Set output mode where MODE is one of:\n" - " ascii Columns/rows delimited by 0x1F and 0x1E\n" - " csv Comma-separated values\n" - " column Left-aligned columns. (See .width)\n" - " html HTML <table> code\n" - " insert SQL insert statements for TABLE\n" - " line One value per line\n" - " list Values delimited by \"|\"\n" - " quote Escape answers as for SQL\n" - " tabs Tab-separated values\n" - " tcl TCL list elements\n" - ".nullvalue STRING Use STRING in place of NULL values\n" - ".once (-e|-x|FILE) Output for the next SQL command only to FILE\n" - " or invoke system text editor (-e) or spreadsheet (-x)\n" - " on the output.\n" - ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE\n" - " The --new option starts with an empty file\n" - ".output ?FILE? Send output to FILE or stdout\n" - ".print STRING... Print literal STRING\n" - ".prompt MAIN CONTINUE Replace the standard prompts\n" - ".quit Exit this program\n" - ".read FILENAME Execute SQL in FILENAME\n" - ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE\n" - ".save FILE Write in-memory database into FILE\n" - ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off\n" - ".schema ?PATTERN? Show the CREATE statements matching PATTERN\n" - " Add --indent for pretty-printing\n" - ".selftest ?--init? Run tests defined in the SELFTEST table\n" - ".separator COL ?ROW? Change the column separator and optionally the row\n" - " separator for both the output mode and .import\n" + ".load FILE ?ENTRY? Load an extension library", +#endif + ".log FILE|off Turn logging on or off. FILE can be stderr/stdout", + ".mode MODE ?TABLE? Set output mode", + " MODE is one of:", + " ascii Columns/rows delimited by 0x1F and 0x1E", + " csv Comma-separated values", + " column Left-aligned columns. (See .width)", + " html HTML <table> code", + " insert SQL insert statements for TABLE", + " line One value per line", + " list Values delimited by \"|\"", + " quote Escape answers as for SQL", + " tabs Tab-separated values", + " tcl TCL list elements", + ".nullvalue STRING Use STRING in place of NULL values", + ".once (-e|-x|FILE) Output for the next SQL command only to FILE", + " If FILE begins with '|' then open as a pipe", + " Other options:", + " -e Invoke system text editor", + " -x Open in a spreadsheet", + ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE", + " Options:", + " --append Use appendvfs to append database to the end of FILE", +#ifdef SQLITE_ENABLE_DESERIALIZE + " --deserialize Load into memory useing sqlite3_deserialize()", + " --hexdb Load the output of \"dbtotxt\" as an in-memory database", + " --maxsize N Maximum size for --hexdb or --deserialized database", +#endif + " --new Initialize FILE to an empty database", + " --readonly Open FILE readonly", + " --zip FILE is a ZIP archive", + ".output ?FILE? Send output to FILE or stdout if FILE is omitted", + " If FILE begins with '|' then open it as a pipe.", + ".parameter CMD ... Manage SQL parameter bindings", + " clear Erase all bindings", + " init Initialize the TEMP table that holds bindings", + " list List the current parameter bindings", + " set PARAMETER VALUE Given SQL parameter PARAMETER a value of VALUE", + " PARAMETER should start with '$', ':', '@', or '?'", + " unset PARAMETER Remove PARAMETER from the binding table", + ".print STRING... Print literal STRING", +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + ".progress N Invoke progress handler after every N opcodes", + " --limit N Interrupt after N progress callbacks", + " --once Do no more than one progress interrupt", + " --quiet|-q No output except at interrupts", + " --reset Reset the count for each input and interrupt", +#endif + ".prompt MAIN CONTINUE Replace the standard prompts", + ".quit Exit this program", + ".read FILE Read input from FILE", + ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", + ".save FILE Write in-memory database into FILE", + ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", + ".schema ?PATTERN? Show the CREATE statements matching PATTERN", + " Options:", + " --indent Try to pretty-print the schema", + ".selftest ?OPTIONS? Run tests defined in the SELFTEST table", + " Options:", + " --init Create a new SELFTEST table", + " -v Verbose output", + ".separator COL ?ROW? Change the column and row separators", #if defined(SQLITE_ENABLE_SESSION) - ".session CMD ... Create or control sessions\n" -#endif - ".sha3sum ?OPTIONS...? Compute a SHA3 hash of database content\n" - ".shell CMD ARGS... Run CMD ARGS... in a system shell\n" - ".show Show the current values for various settings\n" - ".stats ?on|off? Show stats or turn stats on or off\n" - ".system CMD ARGS... Run CMD ARGS... in a system shell\n" - ".tables ?TABLE? List names of tables\n" - " If TABLE specified, only list tables matching\n" - " LIKE pattern TABLE.\n" - ".testcase NAME Begin redirecting output to 'testcase-out.txt'\n" - ".timeout MS Try opening locked tables for MS milliseconds\n" - ".timer on|off Turn SQL timer on or off\n" - ".trace FILE|off Output each SQL statement as it is run\n" - ".vfsinfo ?AUX? Information about the top-level VFS\n" - ".vfslist List all available VFSes\n" - ".vfsname ?AUX? Print the name of the VFS stack\n" - ".width NUM1 NUM2 ... Set column widths for \"column\" mode\n" - " Negative values right-justify\n" -; + ".session ?NAME? CMD ... Create or control sessions", + " Subcommands:", + " attach TABLE Attach TABLE", + " changeset FILE Write a changeset into FILE", + " close Close one session", + " enable ?BOOLEAN? Set or query the enable bit", + " filter GLOB... Reject tables matching GLOBs", + " indirect ?BOOLEAN? Mark or query the indirect status", + " isempty Query whether the session is empty", + " list List currently open session names", + " open DB NAME Open a new session on DB", + " patchset FILE Write a patchset into FILE", + " If ?NAME? is omitted, the first defined session is used.", +#endif + ".sha3sum ... Compute a SHA3 hash of database content", + " Options:", + " --schema Also hash the sqlite_master table", + " --sha3-224 Use the sha3-224 algorithm", + " --sha3-256 Use the sha3-256 algorithm. This is the default.", + " --sha3-384 Use the sha3-384 algorithm", + " --sha3-512 Use the sha3-512 algorithm", + " Any other argument is a LIKE pattern for tables to hash", +#ifndef SQLITE_NOHAVE_SYSTEM + ".shell CMD ARGS... Run CMD ARGS... in a system shell", +#endif + ".show Show the current values for various settings", + ".stats ?on|off? Show stats or turn stats on or off", +#ifndef SQLITE_NOHAVE_SYSTEM + ".system CMD ARGS... Run CMD ARGS... in a system shell", +#endif + ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", + ".testcase NAME Begin redirecting output to 'testcase-out.txt'", + ".timeout MS Try opening locked tables for MS milliseconds", + ".timer on|off Turn SQL timer on or off", +#ifndef SQLITE_OMIT_TRACE + ".trace ?OPTIONS? Output each SQL statement as it is run", + " FILE Send output to FILE", + " stdout Send output to stdout", + " stderr Send output to stderr", + " off Disable tracing", + " --expanded Expand query parameters", +#ifdef SQLITE_ENABLE_NORMALIZE + " --normalized Normal the SQL statements", +#endif + " --plain Show SQL as it is input", + " --stmt Trace statement execution (SQLITE_TRACE_STMT)", + " --profile Profile statements (SQLITE_TRACE_PROFILE)", + " --row Trace each row (SQLITE_TRACE_ROW)", + " --close Trace connection close (SQLITE_TRACE_CLOSE)", +#endif /* SQLITE_OMIT_TRACE */ + ".vfsinfo ?AUX? Information about the top-level VFS", + ".vfslist List all available VFSes", + ".vfsname ?AUX? Print the name of the VFS stack", + ".width NUM1 NUM2 ... Set column widths for \"column\" mode", + " Negative values right-justify", +}; -#if defined(SQLITE_ENABLE_SESSION) /* -** Print help information for the ".sessions" command -*/ -void session_help(ShellState *p){ - raw_printf(p->out, - ".session ?NAME? SUBCOMMAND ?ARGS...?\n" - "If ?NAME? is omitted, the first defined session is used.\n" - "Subcommands:\n" - " attach TABLE Attach TABLE\n" - " changeset FILE Write a changeset into FILE\n" - " close Close one session\n" - " enable ?BOOLEAN? Set or query the enable bit\n" - " filter GLOB... Reject tables matching GLOBs\n" - " indirect ?BOOLEAN? Mark or query the indirect status\n" - " isempty Query whether the session is empty\n" - " list List currently open session names\n" - " open DB NAME Open a new session on DB\n" - " patchset FILE Write a patchset into FILE\n" - ); +** Output help text. +** +** zPattern describes the set of commands for which help text is provided. +** If zPattern is NULL, then show all commands, but only give a one-line +** description of each. +** +** Return the number of matches. +*/ +static int showHelp(FILE *out, const char *zPattern){ + int i = 0; + int j = 0; + int n = 0; + char *zPat; + if( zPattern==0 + || zPattern[0]=='0' + || strcmp(zPattern,"-a")==0 + || strcmp(zPattern,"-all")==0 + ){ + /* Show all commands, but only one line per command */ + if( zPattern==0 ) zPattern = ""; + for(i=0; i<ArraySize(azHelp); i++){ + if( azHelp[i][0]=='.' || zPattern[0] ){ + utf8_printf(out, "%s\n", azHelp[i]); + n++; + } + } + }else{ + /* Look for commands that for which zPattern is an exact prefix */ + zPat = sqlite3_mprintf(".%s*", zPattern); + for(i=0; i<ArraySize(azHelp); i++){ + if( sqlite3_strglob(zPat, azHelp[i])==0 ){ + utf8_printf(out, "%s\n", azHelp[i]); + j = i+1; + n++; + } + } + sqlite3_free(zPat); + if( n ){ + if( n==1 ){ + /* when zPattern is a prefix of exactly one command, then include the + ** details of that command, which should begin at offset j */ + while( j<ArraySize(azHelp)-1 && azHelp[j][0]!='.' ){ + utf8_printf(out, "%s\n", azHelp[j]); + j++; + } + } + return n; + } + /* Look for commands that contain zPattern anywhere. Show the complete + ** text of all commands that match. */ + zPat = sqlite3_mprintf("%%%s%%", zPattern); + for(i=0; i<ArraySize(azHelp); i++){ + if( azHelp[i][0]=='.' ) j = i; + if( sqlite3_strlike(zPat, azHelp[i], 0)==0 ){ + utf8_printf(out, "%s\n", azHelp[j]); + while( j<ArraySize(azHelp)-1 && azHelp[j+1][0]!='.' ){ + j++; + utf8_printf(out, "%s\n", azHelp[j]); + } + i = j; + n++; + } + } + sqlite3_free(zPat); + } + return n; } -#endif - /* Forward reference */ -static int process_input(ShellState *p, FILE *in); +static int process_input(ShellState *p); /* ** Read the content of file zName into memory obtained from sqlite3_malloc64() @@ -10257,7 +11429,7 @@ static char *readFile(const char *zName, int *pnByte){ nIn = ftell(in); rewind(in); pBuf = sqlite3_malloc64( nIn+1 ); - if( pBuf==0 ) return 0; + if( pBuf==0 ){ fclose(in); return 0; } nRead = fread(pBuf, nIn, 1, in); fclose(in); if( nRead!=1 ){ @@ -10319,13 +11491,29 @@ static int session_filter(void *pCtx, const char *zTab){ /* ** Try to deduce the type of file for zName based on its content. Return ** one of the SHELL_OPEN_* constants. +** +** If the file does not exist or is empty but its name looks like a ZIP +** archive and the dfltZip flag is true, then assume it is a ZIP archive. +** Otherwise, assume an ordinary database regardless of the filename if +** the type cannot be determined from content. */ -static int deduceDatabaseType(const char *zName){ +int deduceDatabaseType(const char *zName, int dfltZip){ FILE *f = fopen(zName, "rb"); size_t n; int rc = SHELL_OPEN_UNSPEC; char zBuf[100]; - if( f==0 ) return SHELL_OPEN_NORMAL; + if( f==0 ){ + if( dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){ + return SHELL_OPEN_ZIPFILE; + }else{ + return SHELL_OPEN_NORMAL; + } + } + n = fread(zBuf, 16, 1, f); + if( n==1 && memcmp(zBuf, "SQLite format 3", 16)==0 ){ + fclose(f); + return SHELL_OPEN_NORMAL; + } fseek(f, -25, SEEK_END); n = fread(zBuf, 25, 1, f); if( n==1 && memcmp(zBuf, "Start-Of-SQLite3-", 17)==0 ){ @@ -10336,21 +11524,128 @@ static int deduceDatabaseType(const char *zName){ if( n==1 && zBuf[0]==0x50 && zBuf[1]==0x4b && zBuf[2]==0x05 && zBuf[3]==0x06 ){ rc = SHELL_OPEN_ZIPFILE; + }else if( n==0 && dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){ + rc = SHELL_OPEN_ZIPFILE; } } fclose(f); return rc; } +#ifdef SQLITE_ENABLE_DESERIALIZE +/* +** Reconstruct an in-memory database using the output from the "dbtotxt" +** program. Read content from the file in p->zDbFilename. If p->zDbFilename +** is 0, then read from standard input. +*/ +static unsigned char *readHexDb(ShellState *p, int *pnData){ + unsigned char *a = 0; + int nLine; + int n = 0; + int pgsz = 0; + int iOffset = 0; + int j, k; + int rc; + FILE *in; + unsigned char x[16]; + char zLine[1000]; + if( p->zDbFilename ){ + in = fopen(p->zDbFilename, "r"); + if( in==0 ){ + utf8_printf(stderr, "cannot open \"%s\" for reading\n", p->zDbFilename); + return 0; + } + nLine = 0; + }else{ + in = p->in; + nLine = p->lineno; + } + *pnData = 0; + nLine++; + if( fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error; + rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz); + if( rc!=2 ) goto readHexDb_error; + if( n<=0 ) goto readHexDb_error; + a = sqlite3_malloc( n ); + if( a==0 ){ + utf8_printf(stderr, "Out of memory!\n"); + goto readHexDb_error; + } + memset(a, 0, n); + if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){ + utf8_printf(stderr, "invalid pagesize\n"); + goto readHexDb_error; + } + for(nLine++; fgets(zLine, sizeof(zLine), in)!=0; nLine++){ + rc = sscanf(zLine, "| page %d offset %d", &j, &k); + if( rc==2 ){ + iOffset = k; + continue; + } + if( strncmp(zLine, "| end ", 6)==0 ){ + break; + } + rc = sscanf(zLine,"| %d: %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx" + " %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx", + &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7], + &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]); + if( rc==17 ){ + k = iOffset+j; + if( k+16<=n ){ + memcpy(a+k, x, 16); + } + } + } + *pnData = n; + if( in!=p->in ){ + fclose(in); + }else{ + p->lineno = nLine; + } + return a; + +readHexDb_error: + if( in!=stdin ){ + fclose(in); + }else{ + while( fgets(zLine, sizeof(zLine), p->in)!=0 ){ + nLine++; + if(strncmp(zLine, "| end ", 6)==0 ) break; + } + p->lineno = nLine; + } + sqlite3_free(a); + utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine); + return 0; +} +#endif /* SQLITE_ENABLE_DESERIALIZE */ + +/* Flags for open_db(). +** +** The default behavior of open_db() is to exit(1) if the database fails to +** open. The OPEN_DB_KEEPALIVE flag changes that so that it prints an error +** but still returns without calling exit. +** +** The OPEN_DB_ZIPFILE flag causes open_db() to prefer to open files as a +** ZIP archive if the file does not exist or is empty and its name matches +** the *.zip pattern. +*/ +#define OPEN_DB_KEEPALIVE 0x001 /* Return after error if true */ +#define OPEN_DB_ZIPFILE 0x002 /* Open as ZIP if name matches *.zip */ + /* ** Make sure the database is open. If it is not, then open it. If ** the database fails to open, print an error message and exit. */ -static void open_db(ShellState *p, int keepAlive){ +static void open_db(ShellState *p, int openFlags){ if( p->db==0 ){ - sqlite3_initialize(); - if( p->openMode==SHELL_OPEN_UNSPEC && access(p->zDbFilename,0)==0 ){ - p->openMode = deduceDatabaseType(p->zDbFilename); + if( p->openMode==SHELL_OPEN_UNSPEC ){ + if( p->zDbFilename==0 || p->zDbFilename[0]==0 ){ + p->openMode = SHELL_OPEN_NORMAL; + }else{ + p->openMode = (u8)deduceDatabaseType(p->zDbFilename, + (openFlags & OPEN_DB_ZIPFILE)!=0); + } } switch( p->openMode ){ case SHELL_OPEN_APPENDVFS: { @@ -10358,10 +11653,19 @@ static void open_db(ShellState *p, int keepAlive){ SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs"); break; } + case SHELL_OPEN_HEXDB: + case SHELL_OPEN_DESERIALIZE: { + sqlite3_open(0, &p->db); + break; + } case SHELL_OPEN_ZIPFILE: { sqlite3_open(":memory:", &p->db); break; } + case SHELL_OPEN_READONLY: { + sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READONLY, 0); + break; + } case SHELL_OPEN_UNSPEC: case SHELL_OPEN_NORMAL: { sqlite3_open(p->zDbFilename, &p->db); @@ -10372,7 +11676,10 @@ static void open_db(ShellState *p, int keepAlive){ if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", p->zDbFilename, sqlite3_errmsg(p->db)); - if( keepAlive ) return; + if( openFlags & OPEN_DB_KEEPALIVE ){ + sqlite3_open(":memory:", &p->db); + return; + } exit(1); } #ifndef SQLITE_OMIT_LOAD_EXTENSION @@ -10391,26 +11698,28 @@ static void open_db(ShellState *p, int keepAlive){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); +#ifndef SQLITE_NOHAVE_SYSTEM sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, editFunc, 0, 0); sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, editFunc, 0, 0); +#endif - // Begin Android Add - #ifndef NO_ANDROID_FUNCS - AIcu_initializeIcuOrDie(); - int err = register_localized_collators(p->db, "en_US", 0); - if (err != SQLITE_OK) { - fprintf(stderr, "register_localized_collators() failed\n"); - exit(1); - } - err = register_android_functions(p->db, 0); - if (err != SQLITE_OK) { - fprintf(stderr, "register_android_functions() failed\n"); - exit(1); - } - #endif - // End Android Add +// Begin Android Add +#ifndef NO_ANDROID_FUNCS + AIcu_initializeIcuOrDie(); + int err = register_localized_collators(p->db, "en_US", 0); + if (err != SQLITE_OK) { + fprintf(stderr, "register_localized_collators() failed\n"); + exit(1); + } + err = register_android_functions(p->db, 0); + if (err != SQLITE_OK) { + fprintf(stderr, "register_android_functions() failed\n"); + exit(1); + } +#endif +// End Android Add if( p->openMode==SHELL_OPEN_ZIPFILE ){ char *zSql = sqlite3_mprintf( @@ -10418,9 +11727,46 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_free(zSql); } +#ifdef SQLITE_ENABLE_DESERIALIZE + else + if( p->openMode==SHELL_OPEN_DESERIALIZE || p->openMode==SHELL_OPEN_HEXDB ){ + int rc; + int nData = 0; + unsigned char *aData; + if( p->openMode==SHELL_OPEN_DESERIALIZE ){ + aData = (unsigned char*)readFile(p->zDbFilename, &nData); + }else{ + aData = readHexDb(p, &nData); + if( aData==0 ){ + utf8_printf(stderr, "Error in hexdb input\n"); + return; + } + } + rc = sqlite3_deserialize(p->db, "main", aData, nData, nData, + SQLITE_DESERIALIZE_RESIZEABLE | + SQLITE_DESERIALIZE_FREEONCLOSE); + if( rc ){ + utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc); + } + if( p->szMax>0 ){ + sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax); + } + } +#endif } } +/* +** Attempt to close the databaes connection. Report errors. +*/ +void close_db(sqlite3 *db){ + int rc = sqlite3_close(db); + if( rc ){ + utf8_printf(stderr, "Error: sqlite3_close() returns %d: %s\n", + rc, sqlite3_errmsg(db)); + } +} + #if HAVE_READLINE || HAVE_EDITLINE /* ** Readline completion callbacks @@ -10462,7 +11808,7 @@ static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){ char zBuf[1000]; if( nLine>sizeof(zBuf)-30 ) return; - if( zLine[0]=='.' ) return; + if( zLine[0]=='.' || zLine[0]=='#') return; for(i=nLine-1; i>=0 && (isalnum(zLine[i]) || zLine[i]=='_'); i--){} if( i==nLine-1 ) return; iStart = i+1; @@ -10608,30 +11954,64 @@ static FILE *output_file_open(const char *zFile, int bTextMode){ return f; } -#if !defined(SQLITE_UNTESTABLE) -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) +#ifndef SQLITE_OMIT_TRACE /* ** A routine for handling output from sqlite3_trace(). */ static int sql_trace_callback( - unsigned mType, - void *pArg, - void *pP, - void *pX + unsigned mType, /* The trace type */ + void *pArg, /* The ShellState pointer */ + void *pP, /* Usually a pointer to sqlite_stmt */ + void *pX /* Auxiliary output */ ){ - FILE *f = (FILE*)pArg; - UNUSED_PARAMETER(mType); - UNUSED_PARAMETER(pP); - if( f ){ - const char *z = (const char*)pX; - int i = strlen30(z); - while( i>0 && z[i-1]==';' ){ i--; } - utf8_printf(f, "%.*s;\n", i, z); + ShellState *p = (ShellState*)pArg; + sqlite3_stmt *pStmt; + const char *zSql; + int nSql; + if( p->traceOut==0 ) return 0; + if( mType==SQLITE_TRACE_CLOSE ){ + utf8_printf(p->traceOut, "-- closing database connection\n"); + return 0; + } + if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){ + zSql = (const char*)pX; + }else{ + pStmt = (sqlite3_stmt*)pP; + switch( p->eTraceType ){ + case SHELL_TRACE_EXPANDED: { + zSql = sqlite3_expanded_sql(pStmt); + break; + } +#ifdef SQLITE_ENABLE_NORMALIZE + case SHELL_TRACE_NORMALIZED: { + zSql = sqlite3_normalized_sql(pStmt); + break; + } +#endif + default: { + zSql = sqlite3_sql(pStmt); + break; + } + } + } + if( zSql==0 ) return 0; + nSql = strlen30(zSql); + while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; } + switch( mType ){ + case SQLITE_TRACE_ROW: + case SQLITE_TRACE_STMT: { + utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql); + break; + } + case SQLITE_TRACE_PROFILE: { + sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; + utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec); + break; + } } return 0; } #endif -#endif /* ** A no-op routine that runs with the ".breakpoint" doc-command. This is @@ -10664,10 +12044,7 @@ static void import_append_char(ImportCtx *p, int c){ if( p->n+1>=p->nAlloc ){ p->nAlloc += p->nAlloc + 100; p->z = sqlite3_realloc64(p->z, p->nAlloc); - if( p->z==0 ){ - raw_printf(stderr, "out of memory\n"); - exit(1); - } + if( p->z==0 ) shell_out_of_memory(); } p->z[p->n++] = (char)c; } @@ -10828,10 +12205,7 @@ static void tryToCloneData( } n = sqlite3_column_count(pQuery); zInsert = sqlite3_malloc64(200 + nTable + n*3); - if( zInsert==0 ){ - raw_printf(stderr, "out of memory\n"); - goto end_data_xfer; - } + if( zInsert==0 ) shell_out_of_memory(); sqlite3_snprintf(200+nTable,zInsert, "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable); i = strlen30(zInsert); @@ -11009,7 +12383,7 @@ static void tryToClone(ShellState *p, const char *zNewDb){ sqlite3_exec(newDb, "COMMIT;", 0, 0, 0); sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); } - sqlite3_close(newDb); + close_db(newDb); } /* @@ -11026,6 +12400,7 @@ static void output_reset(ShellState *p){ #endif }else{ output_file_close(p->out); +#ifndef SQLITE_NOHAVE_SYSTEM if( p->doXdgOpen ){ const char *zXdgOpenCmd = #if defined(_WIN32) @@ -11044,6 +12419,7 @@ static void output_reset(ShellState *p){ outputModePop(p); p->doXdgOpen = 0; } +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ } p->outfile[0] = 0; p->out = stdout; @@ -11105,15 +12481,27 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ { "schema size:", "SELECT total(length(sql)) FROM %s" }, }; - int i; + int i, rc; + unsigned iDataVersion; char *zSchemaTab; char *zDb = nArg>=2 ? azArg[1] : "main"; sqlite3_stmt *pStmt = 0; unsigned char aHdr[100]; open_db(p, 0); if( p->db==0 ) return 1; - sqlite3_prepare_v2(p->db,"SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", - -1, &pStmt, 0); + rc = sqlite3_prepare_v2(p->db, + "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", + -1, &pStmt, 0); + if( rc ){ + if( !sqlite3_compileoption_used("ENABLE_DBPAGE_VTAB") ){ + utf8_printf(stderr, "the \".dbinfo\" command requires the " + "-DSQLITE_ENABLE_DBPAGE_VTAB compile-time options\n"); + }else{ + utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); + } + sqlite3_finalize(pStmt); + return 1; + } sqlite3_bind_text(pStmt, 1, zDb, -1, SQLITE_STATIC); if( sqlite3_step(pStmt)==SQLITE_ROW && sqlite3_column_bytes(pStmt,0)>100 @@ -11158,6 +12546,8 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ utf8_printf(p->out, "%-20s %d\n", aQuery[i].zName, val); } sqlite3_free(zSchemaTab); + sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion); + utf8_printf(p->out, "%-20s %u\n", "data version", iDataVersion); return 0; } @@ -11171,14 +12561,6 @@ static int shellDatabaseError(sqlite3 *db){ } /* -** Print an out-of-memory message to stderr and return 1. -*/ -static int shellNomemError(void){ - raw_printf(stderr, "Error: out of memory\n"); - return 1; -} - -/* ** Compare the pattern in zGlob[] against the text in z[]. Return TRUE ** if they match and FALSE (0) if they do not match. ** @@ -11622,6 +13004,7 @@ static void shellPreparePrintf( char *z; va_start(ap, zFmt); z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); if( z==0 ){ *pRc = SQLITE_NOMEM; }else{ @@ -11670,6 +13053,7 @@ struct ArCommand { u8 bZip; /* True if the archive is a ZIP */ u8 bDryRun; /* True if --dry-run */ u8 bAppend; /* True if --append */ + u8 fromCmdLine; /* Run from -A instead of .archive */ int nArg; /* Number of command arguments */ char *zSrcTable; /* "sqlar", "zipfile($file)" or "zip" */ const char *zFile; /* --file argument, or NULL */ @@ -11683,32 +13067,7 @@ struct ArCommand { ** Print a usage message for the .ar command to stderr and return SQLITE_ERROR. */ static int arUsage(FILE *f){ - raw_printf(f, -"\n" -"Usage: .ar [OPTION...] [FILE...]\n" -"The .ar command manages sqlar archives.\n" -"\n" -"Examples:\n" -" .ar -cf archive.sar foo bar # Create archive.sar from files foo and bar\n" -" .ar -tf archive.sar # List members of archive.sar\n" -" .ar -xvf archive.sar # Verbosely extract files from archive.sar\n" -"\n" -"Each command line must feature exactly one command option:\n" -" -c, --create Create a new archive\n" -" -u, --update Update or add files to an existing archive\n" -" -t, --list List contents of archive\n" -" -x, --extract Extract files from archive\n" -"\n" -"And zero or more optional options:\n" -" -v, --verbose Print each filename as it is processed\n" -" -f FILE, --file FILE Operate on archive FILE (default is current db)\n" -" -a FILE, --append FILE Operate on FILE opened using the apndvfs VFS\n" -" -C DIR, --directory DIR Change to directory DIR to read/extract files\n" -" -n, --dryrun Show the SQL that would have occurred\n" -"\n" -"See also: http://sqlite.org/cli.html#sqlar_archive_support\n" -"\n" -); + showHelp(f,"archive"); return SQLITE_ERROR; } @@ -11716,13 +13075,18 @@ static int arUsage(FILE *f){ ** Print an error message for the .ar command to stderr and return ** SQLITE_ERROR. */ -static int arErrorMsg(const char *zFmt, ...){ +static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ va_list ap; char *z; va_start(ap, zFmt); z = sqlite3_vmprintf(zFmt, ap); va_end(ap); - raw_printf(stderr, "Error: %s (try \".ar --help\")\n", z); + utf8_printf(stderr, "Error: %s\n", z); + if( pAr->fromCmdLine ){ + utf8_printf(stderr, "Use \"-A\" for more help\n"); + }else{ + utf8_printf(stderr, "Use \".archive --help\" for more help\n"); + } sqlite3_free(z); return SQLITE_ERROR; } @@ -11731,19 +13095,20 @@ static int arErrorMsg(const char *zFmt, ...){ ** Values for ArCommand.eCmd. */ #define AR_CMD_CREATE 1 -#define AR_CMD_EXTRACT 2 -#define AR_CMD_LIST 3 -#define AR_CMD_UPDATE 4 -#define AR_CMD_HELP 5 +#define AR_CMD_UPDATE 2 +#define AR_CMD_INSERT 3 +#define AR_CMD_EXTRACT 4 +#define AR_CMD_LIST 5 +#define AR_CMD_HELP 6 /* ** Other (non-command) switches. */ -#define AR_SWITCH_VERBOSE 6 -#define AR_SWITCH_FILE 7 -#define AR_SWITCH_DIRECTORY 8 -#define AR_SWITCH_APPEND 9 -#define AR_SWITCH_DRYRUN 10 +#define AR_SWITCH_VERBOSE 7 +#define AR_SWITCH_FILE 8 +#define AR_SWITCH_DIRECTORY 9 +#define AR_SWITCH_APPEND 10 +#define AR_SWITCH_DRYRUN 11 static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){ switch( eSwitch ){ @@ -11751,9 +13116,10 @@ static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){ case AR_CMD_EXTRACT: case AR_CMD_LIST: case AR_CMD_UPDATE: + case AR_CMD_INSERT: case AR_CMD_HELP: if( pAr->eCmd ){ - return arErrorMsg("multiple command options"); + return arErrorMsg(pAr, "multiple command options"); } pAr->eCmd = eSwitch; break; @@ -11797,6 +13163,7 @@ static int arParseCommand( } aSwitch[] = { { "create", 'c', AR_CMD_CREATE, 0 }, { "extract", 'x', AR_CMD_EXTRACT, 0 }, + { "insert", 'i', AR_CMD_INSERT, 0 }, { "list", 't', AR_CMD_LIST, 0 }, { "update", 'u', AR_CMD_UPDATE, 0 }, { "help", 'h', AR_CMD_HELP, 0 }, @@ -11810,11 +13177,10 @@ static int arParseCommand( struct ArSwitch *pEnd = &aSwitch[nSwitch]; if( nArg<=1 ){ + utf8_printf(stderr, "Wrong number of arguments. Usage:\n"); return arUsage(stderr); }else{ char *z = azArg[1]; - memset(pAr, 0, sizeof(ArCommand)); - if( z[0]!='-' ){ /* Traditional style [tar] invocation */ int i; @@ -11826,11 +13192,11 @@ static int arParseCommand( if( z[i]==pOpt->cShort ) break; } if( pOpt==pEnd ){ - return arErrorMsg("unrecognized option: %c", z[i]); + return arErrorMsg(pAr, "unrecognized option: %c", z[i]); } if( pOpt->bArg ){ if( iArg>=nArg ){ - return arErrorMsg("option requires an argument: %c",z[i]); + return arErrorMsg(pAr, "option requires an argument: %c",z[i]); } zArg = azArg[iArg++]; } @@ -11864,7 +13230,7 @@ static int arParseCommand( if( z[i]==pOpt->cShort ) break; } if( pOpt==pEnd ){ - return arErrorMsg("unrecognized option: %c\n", z[i]); + return arErrorMsg(pAr, "unrecognized option: %c", z[i]); } if( pOpt->bArg ){ if( i<(n-1) ){ @@ -11872,7 +13238,7 @@ static int arParseCommand( i = n; }else{ if( iArg>=(nArg-1) ){ - return arErrorMsg("option requires an argument: %c\n",z[i]); + return arErrorMsg(pAr, "option requires an argument: %c",z[i]); } zArg = azArg[++iArg]; } @@ -11894,7 +13260,7 @@ static int arParseCommand( const char *zLong = pOpt->zLong; if( (n-2)<=strlen30(zLong) && 0==memcmp(&z[2], zLong, n-2) ){ if( pMatch ){ - return arErrorMsg("ambiguous option: %s",z); + return arErrorMsg(pAr, "ambiguous option: %s",z); }else{ pMatch = pOpt; } @@ -11902,11 +13268,11 @@ static int arParseCommand( } if( pMatch==0 ){ - return arErrorMsg("unrecognized option: %s", z); + return arErrorMsg(pAr, "unrecognized option: %s", z); } if( pMatch->bArg ){ if( iArg>=(nArg-1) ){ - return arErrorMsg("option requires an argument: %s", z); + return arErrorMsg(pAr, "option requires an argument: %s", z); } zArg = azArg[++iArg]; } @@ -12035,6 +13401,7 @@ static int arListCommand(ArCommand *pAr){ } } shellFinalize(&rc, pSql); + sqlite3_free(zWhere); return rc; } @@ -12047,7 +13414,8 @@ static int arExtractCommand(ArCommand *pAr){ "SELECT " " ($dir || name)," " writefile(($dir || name), %s, mode, mtime) " - "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)"; + "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)" + " AND name NOT GLOB '*..[/\\]*'"; const char *azExtraArg[] = { "sqlar_uncompress(data, sz)", @@ -12131,7 +13499,12 @@ static int arExecSql(ArCommand *pAr, const char *zSql){ /* -** Implementation of .ar "create" and "update" commands. +** Implementation of .ar "create", "insert", and "update" commands. +** +** create -> Create a new SQL archive +** insert -> Insert or reinsert all files listed +** update -> Insert files that have changed or that were not +** previously in the archive ** ** Create the "sqlar" table in the database if it does not already exist. ** Then add each file in the azFile[] array to the archive. Directories @@ -12139,11 +13512,14 @@ static int arExecSql(ArCommand *pAr, const char *zSql){ ** printed on stdout for each file archived. ** ** The create command is the same as update, except that it drops -** any existing "sqlar" table before beginning. +** any existing "sqlar" table before beginning. The "insert" command +** always overwrites every file named on the command-line, where as +** "update" only overwrites if the size or mtime or mode has changed. */ static int arCreateOrUpdateCommand( ArCommand *pAr, /* Command arguments and options */ - int bUpdate /* true for a --create. false for --update */ + int bUpdate, /* true for a --create. */ + int bOnlyIfChanged /* Only update if file has changed */ ){ const char *zCreate = "CREATE TABLE IF NOT EXISTS sqlar(\n" @@ -12154,8 +13530,8 @@ static int arCreateOrUpdateCommand( " data BLOB -- compressed content\n" ")"; const char *zDrop = "DROP TABLE IF EXISTS sqlar"; - const char *zInsertFmt = - "REPLACE INTO sqlar(name,mode,mtime,sz,data)\n" + const char *zInsertFmt[2] = { + "REPLACE INTO %s(name,mode,mtime,sz,data)\n" " SELECT\n" " %s,\n" " mode,\n" @@ -12164,31 +13540,85 @@ static int arCreateOrUpdateCommand( " WHEN '-' THEN length(data)\n" " WHEN 'd' THEN 0\n" " ELSE -1 END,\n" - " CASE WHEN lsmode(mode) LIKE 'd%%' THEN NULL else data END\n" - " FROM fsdir(%Q,%Q)\n" - " WHERE lsmode(mode) NOT LIKE '?%%';"; + " sqlar_compress(data)\n" + " FROM fsdir(%Q,%Q) AS disk\n" + " WHERE lsmode(mode) NOT LIKE '?%%'%s;" + , + "REPLACE INTO %s(name,mode,mtime,data)\n" + " SELECT\n" + " %s,\n" + " mode,\n" + " mtime,\n" + " data\n" + " FROM fsdir(%Q,%Q) AS disk\n" + " WHERE lsmode(mode) NOT LIKE '?%%'%s;" + }; int i; /* For iterating through azFile[] */ int rc; /* Return code */ + const char *zTab = 0; /* SQL table into which to insert */ + char *zSql; + char zTemp[50]; + char *zExists = 0; + arExecSql(pAr, "PRAGMA page_size=512"); rc = arExecSql(pAr, "SAVEPOINT ar;"); if( rc!=SQLITE_OK ) return rc; - if( bUpdate==0 ){ - rc = arExecSql(pAr, zDrop); - if( rc!=SQLITE_OK ) return rc; + zTemp[0] = 0; + if( pAr->bZip ){ + /* Initialize the zipfile virtual table, if necessary */ + if( pAr->zFile ){ + sqlite3_uint64 r; + sqlite3_randomness(sizeof(r),&r); + sqlite3_snprintf(sizeof(zTemp),zTemp,"zip%016llx",r); + zTab = zTemp; + zSql = sqlite3_mprintf( + "CREATE VIRTUAL TABLE temp.%s USING zipfile(%Q)", + zTab, pAr->zFile + ); + rc = arExecSql(pAr, zSql); + sqlite3_free(zSql); + }else{ + zTab = "zip"; + } + }else{ + /* Initialize the table for an SQLAR */ + zTab = "sqlar"; + if( bUpdate==0 ){ + rc = arExecSql(pAr, zDrop); + if( rc!=SQLITE_OK ) goto end_ar_transaction; + } + rc = arExecSql(pAr, zCreate); + } + if( bOnlyIfChanged ){ + zExists = sqlite3_mprintf( + " AND NOT EXISTS(" + "SELECT 1 FROM %s AS mem" + " WHERE mem.name=disk.name" + " AND mem.mtime=disk.mtime" + " AND mem.mode=disk.mode)", zTab); + }else{ + zExists = sqlite3_mprintf(""); } - rc = arExecSql(pAr, zCreate); + if( zExists==0 ) rc = SQLITE_NOMEM; for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){ - char *zSql = sqlite3_mprintf(zInsertFmt, + char *zSql2 = sqlite3_mprintf(zInsertFmt[pAr->bZip], zTab, pAr->bVerbose ? "shell_putsnl(name)" : "name", - pAr->azArg[i], pAr->zDir); - rc = arExecSql(pAr, zSql); - sqlite3_free(zSql); + pAr->azArg[i], pAr->zDir, zExists); + rc = arExecSql(pAr, zSql2); + sqlite3_free(zSql2); } +end_ar_transaction: if( rc!=SQLITE_OK ){ - arExecSql(pAr, "ROLLBACK TO ar; RELEASE ar;"); + sqlite3_exec(pAr->db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0); }else{ rc = arExecSql(pAr, "RELEASE ar;"); + if( pAr->bZip && pAr->zFile ){ + zSql = sqlite3_mprintf("DROP TABLE %s", zTemp); + arExecSql(pAr, zSql); + sqlite3_free(zSql); + } } + sqlite3_free(zExists); return rc; } @@ -12197,38 +13627,38 @@ static int arCreateOrUpdateCommand( */ static int arDotCommand( ShellState *pState, /* Current shell tool state */ + int fromCmdLine, /* True if -A command-line option, not .ar cmd */ char **azArg, /* Array of arguments passed to dot command */ int nArg /* Number of entries in azArg[] */ ){ ArCommand cmd; int rc; memset(&cmd, 0, sizeof(cmd)); + cmd.fromCmdLine = fromCmdLine; rc = arParseCommand(azArg, nArg, &cmd); if( rc==SQLITE_OK ){ int eDbType = SHELL_OPEN_UNSPEC; cmd.p = pState; cmd.db = pState->db; if( cmd.zFile ){ - eDbType = deduceDatabaseType(cmd.zFile); + eDbType = deduceDatabaseType(cmd.zFile, 1); }else{ eDbType = pState->openMode; } if( eDbType==SHELL_OPEN_ZIPFILE ){ - if( cmd.zFile==0 ){ - cmd.zSrcTable = sqlite3_mprintf("zip"); - }else{ - cmd.zSrcTable = sqlite3_mprintf("zipfile(%Q)", cmd.zFile); - } - if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_UPDATE ){ - utf8_printf(stderr, "zip archives are read-only\n"); - rc = SQLITE_ERROR; - goto end_ar_command; + if( cmd.eCmd==AR_CMD_EXTRACT || cmd.eCmd==AR_CMD_LIST ){ + if( cmd.zFile==0 ){ + cmd.zSrcTable = sqlite3_mprintf("zip"); + }else{ + cmd.zSrcTable = sqlite3_mprintf("zipfile(%Q)", cmd.zFile); + } } cmd.bZip = 1; }else if( cmd.zFile ){ int flags; if( cmd.bAppend ) eDbType = SHELL_OPEN_APPENDVFS; - if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_UPDATE ){ + if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_INSERT + || cmd.eCmd==AR_CMD_UPDATE ){ flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; }else{ flags = SQLITE_OPEN_READONLY; @@ -12247,14 +13677,12 @@ static int arDotCommand( goto end_ar_command; } sqlite3_fileio_init(cmd.db, 0, 0); -#ifdef SQLITE_HAVE_ZLIB sqlite3_sqlar_init(cmd.db, 0, 0); -#endif sqlite3_create_function(cmd.db, "shell_putsnl", 1, SQLITE_UTF8, cmd.p, shellPutsFunc, 0, 0); } - if( cmd.zSrcTable==0 ){ + if( cmd.zSrcTable==0 && cmd.bZip==0 && cmd.eCmd!=AR_CMD_HELP ){ if( cmd.eCmd!=AR_CMD_CREATE && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) ){ @@ -12267,7 +13695,7 @@ static int arDotCommand( switch( cmd.eCmd ){ case AR_CMD_CREATE: - rc = arCreateOrUpdateCommand(&cmd, 0); + rc = arCreateOrUpdateCommand(&cmd, 0, 0); break; case AR_CMD_EXTRACT: @@ -12282,15 +13710,19 @@ static int arDotCommand( arUsage(pState->out); break; + case AR_CMD_INSERT: + rc = arCreateOrUpdateCommand(&cmd, 1, 0); + break; + default: assert( cmd.eCmd==AR_CMD_UPDATE ); - rc = arCreateOrUpdateCommand(&cmd, 1); + rc = arCreateOrUpdateCommand(&cmd, 1, 1); break; } } end_ar_command: if( cmd.db!=pState->db ){ - sqlite3_close(cmd.db); + close_db(cmd.db); } sqlite3_free(cmd.zSrcTable); @@ -12370,7 +13802,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){ open_db(p, 0); - rc = arDotCommand(p, azArg, nArg); + rc = arDotCommand(p, 0, azArg, nArg); }else #endif @@ -12382,11 +13814,18 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3 *pDest; sqlite3_backup *pBackup; int j; + int bAsync = 0; + const char *zVfs = 0; for(j=1; j<nArg; j++){ const char *z = azArg[j]; if( z[0]=='-' ){ - while( z[0]=='-' ) z++; - /* No options to process at this time */ + if( z[1]=='-' ) z++; + if( strcmp(z, "-append")==0 ){ + zVfs = "apndvfs"; + }else + if( strcmp(z, "-async")==0 ){ + bAsync = 1; + }else { utf8_printf(stderr, "unknown option: %s\n", azArg[j]); return 1; @@ -12397,7 +13836,7 @@ static int do_meta_command(char *zLine, ShellState *p){ zDb = zDestFile; zDestFile = azArg[j]; }else{ - raw_printf(stderr, "too many arguments to .backup\n"); + raw_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); return 1; } } @@ -12406,17 +13845,22 @@ static int do_meta_command(char *zLine, ShellState *p){ return 1; } if( zDb==0 ) zDb = "main"; - rc = sqlite3_open(zDestFile, &pDest); + rc = sqlite3_open_v2(zDestFile, &pDest, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs); if( rc!=SQLITE_OK ){ utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile); - sqlite3_close(pDest); + close_db(pDest); return 1; } + if( bAsync ){ + sqlite3_exec(pDest, "PRAGMA synchronous=OFF; PRAGMA journal_mode=OFF;", + 0, 0, 0); + } open_db(p, 0); pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb); if( pBackup==0 ){ utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); - sqlite3_close(pDest); + close_db(pDest); return 1; } while( (rc = sqlite3_backup_step(pBackup,100))==SQLITE_OK ){} @@ -12427,7 +13871,7 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); rc = 1; } - sqlite3_close(pDest); + close_db(pDest); }else if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 ){ @@ -12539,7 +13983,39 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='d' && strncmp(azArg[0], "dbinfo", n)==0 ){ + if( c=='d' && n>=3 && strncmp(azArg[0], "dbconfig", n)==0 ){ + static const struct DbConfigChoices { + const char *zName; + int op; + } aDbConfig[] = { + { "enable_fkey", SQLITE_DBCONFIG_ENABLE_FKEY }, + { "enable_trigger", SQLITE_DBCONFIG_ENABLE_TRIGGER }, + { "fts3_tokenizer", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, + { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, + { "no_ckpt_on_close", SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE }, + { "enable_qpsg", SQLITE_DBCONFIG_ENABLE_QPSG }, + { "trigger_eqp", SQLITE_DBCONFIG_TRIGGER_EQP }, + { "reset_database", SQLITE_DBCONFIG_RESET_DATABASE }, + { "defensive", SQLITE_DBCONFIG_DEFENSIVE }, + }; + int ii, v; + open_db(p, 0); + for(ii=0; ii<ArraySize(aDbConfig); ii++){ + if( nArg>1 && strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; + if( nArg>=3 ){ + sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); + } + sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v); + utf8_printf(p->out, "%18s %s\n", aDbConfig[ii].zName, v ? "on" : "off"); + if( nArg>1 ) break; + } + if( nArg>1 && ii==ArraySize(aDbConfig) ){ + utf8_printf(stderr, "Error: unknown dbconfig \"%s\"\n", azArg[1]); + utf8_printf(stderr, "Enter \".dbconfig\" with no arguments for a list\n"); + } + }else + + if( c=='d' && n>=3 && strncmp(azArg[0], "dbinfo", n)==0 ){ rc = shell_dbinfo_command(p, nArg, azArg); }else @@ -12547,7 +14023,8 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zLike = 0; int i; int savedShowHeader = p->showHeader; - ShellClearFlag(p, SHFLG_PreserveRowid|SHFLG_Newlines); + int savedShellFlags = p->shellFlgs; + ShellClearFlag(p, SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo); for(i=1; i<nArg; i++){ if( azArg[i][0]=='-' ){ const char *z = azArg[i]+1; @@ -12629,6 +14106,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); raw_printf(p->out, p->nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n"); p->showHeader = savedShowHeader; + p->shellFlgs = savedShellFlags; }else if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){ @@ -12642,15 +14120,31 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){ if( nArg==2 ){ + p->autoEQPtest = 0; + if( p->autoEQPtrace ){ + if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0); + p->autoEQPtrace = 0; + } if( strcmp(azArg[1],"full")==0 ){ p->autoEQP = AUTOEQP_full; }else if( strcmp(azArg[1],"trigger")==0 ){ p->autoEQP = AUTOEQP_trigger; +#ifdef SQLITE_DEBUG + }else if( strcmp(azArg[1],"test")==0 ){ + p->autoEQP = AUTOEQP_on; + p->autoEQPtest = 1; + }else if( strcmp(azArg[1],"trace")==0 ){ + p->autoEQP = AUTOEQP_full; + p->autoEQPtrace = 1; + open_db(p, 0); + sqlite3_exec(p->db, "SELECT name FROM sqlite_master LIMIT 1", 0, 0, 0); + sqlite3_exec(p->db, "PRAGMA vdbe_trace=ON;", 0, 0, 0); +#endif }else{ - p->autoEQP = booleanValue(azArg[1]); + p->autoEQP = (u8)booleanValue(azArg[1]); } }else{ - raw_printf(stderr, "Usage: .eqp off|on|trigger|full\n"); + raw_printf(stderr, "Usage: .eqp off|on|trace|trigger|full\n"); rc = 1; } }else @@ -12734,14 +14228,11 @@ static int do_meta_command(char *zLine, ShellState *p){ callback, &data, &zErrMsg); data.cMode = data.mode = MODE_Insert; data.zDestTable = "sqlite_stat1"; - shell_exec(p->db, "SELECT * FROM sqlite_stat1", - shell_callback, &data,&zErrMsg); + shell_exec(&data, "SELECT * FROM sqlite_stat1", &zErrMsg); data.zDestTable = "sqlite_stat3"; - shell_exec(p->db, "SELECT * FROM sqlite_stat3", - shell_callback, &data,&zErrMsg); + shell_exec(&data, "SELECT * FROM sqlite_stat3", &zErrMsg); data.zDestTable = "sqlite_stat4"; - shell_exec(p->db, "SELECT * FROM sqlite_stat4", - shell_callback, &data, &zErrMsg); + shell_exec(&data, "SELECT * FROM sqlite_stat4", &zErrMsg); raw_printf(p->out, "ANALYZE sqlite_master;\n"); } }else @@ -12756,7 +14247,14 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='h' && strncmp(azArg[0], "help", n)==0 ){ - utf8_printf(p->out, "%s", zHelp); + if( nArg>=2 ){ + n = showHelp(p->out, azArg[1]); + if( n==0 ){ + utf8_printf(p->out, "Nothing matches '%s'\n", azArg[1]); + } + }else{ + showHelp(p->out, 0); + } }else if( c=='i' && strncmp(azArg[0], "import", n)==0 ){ @@ -12839,9 +14337,8 @@ static int do_meta_command(char *zLine, ShellState *p){ sCtx.cRowSep = p->rowSeparator[0]; zSql = sqlite3_mprintf("SELECT * FROM %s", zTable); if( zSql==0 ){ - raw_printf(stderr, "Error: out of memory\n"); xCloser(sCtx.in); - return 1; + shell_out_of_memory(); } nByte = strlen30(zSql); rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); @@ -12886,9 +14383,8 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nCol==0 ) return 0; /* no columns, no error */ zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 ); if( zSql==0 ){ - raw_printf(stderr, "Error: out of memory\n"); xCloser(sCtx.in); - return 1; + shell_out_of_memory(); } sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); j = strlen30(zSql); @@ -12964,12 +14460,17 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_stmt *pStmt; int tnum = 0; int i; - if( nArg!=3 ){ - utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n"); + if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){ + utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n" + " .imposter off\n"); rc = 1; goto meta_command_exit; } open_db(p, 0); + if( nArg==2 ){ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1); + goto meta_command_exit; + } zSql = sqlite3_mprintf("SELECT rootpage FROM sqlite_master" " WHERE name='%q' AND type='index'", azArg[1]); sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); @@ -13211,23 +14712,34 @@ static int do_meta_command(char *zLine, ShellState *p){ int newFlag = 0; /* True to delete file before opening */ /* Close the existing database */ session_close_all(p); - sqlite3_close(p->db); + close_db(p->db); p->db = 0; p->zDbFilename = 0; sqlite3_free(p->zFreeOnClose); p->zFreeOnClose = 0; p->openMode = SHELL_OPEN_UNSPEC; + p->szMax = 0; /* Check for command-line arguments */ for(iName=1; iName<nArg && azArg[iName][0]=='-'; iName++){ const char *z = azArg[iName]; if( optionMatch(z,"new") ){ newFlag = 1; -#ifdef SQLITE_HAVE_ZIP +#ifdef SQLITE_HAVE_ZLIB }else if( optionMatch(z, "zip") ){ p->openMode = SHELL_OPEN_ZIPFILE; #endif }else if( optionMatch(z, "append") ){ p->openMode = SHELL_OPEN_APPENDVFS; + }else if( optionMatch(z, "readonly") ){ + p->openMode = SHELL_OPEN_READONLY; +#ifdef SQLITE_ENABLE_DESERIALIZE + }else if( optionMatch(z, "deserialize") ){ + p->openMode = SHELL_OPEN_DESERIALIZE; + }else if( optionMatch(z, "hexdb") ){ + p->openMode = SHELL_OPEN_HEXDB; + }else if( optionMatch(z, "maxsize") && iName+1<nArg ){ + p->szMax = integerValue(azArg[++iName]); +#endif /* SQLITE_ENABLE_DESERIALIZE */ }else if( z[0]=='-' ){ utf8_printf(stderr, "unknown option: %s\n", z); rc = 1; @@ -13236,10 +14748,10 @@ static int do_meta_command(char *zLine, ShellState *p){ } /* If a filename is specified, try to open it first */ zNewFilename = nArg>iName ? sqlite3_mprintf("%s", azArg[iName]) : 0; - if( zNewFilename ){ + if( zNewFilename || p->openMode==SHELL_OPEN_HEXDB ){ if( newFlag ) shellDeleteFile(zNewFilename); p->zDbFilename = zNewFilename; - open_db(p, 1); + open_db(p, OPEN_DB_KEEPALIVE); if( p->db==0 ){ utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename); sqlite3_free(zNewFilename); @@ -13284,6 +14796,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } output_reset(p); if( zFile[0]=='-' && zFile[1]=='-' ) zFile++; +#ifndef SQLITE_NOHAVE_SYSTEM if( strcmp(zFile, "-e")==0 || strcmp(zFile, "-x")==0 ){ p->doXdgOpen = 1; outputModePush(p); @@ -13298,6 +14811,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } zFile = p->zTempFile; } +#endif /* SQLITE_NOHAVE_SYSTEM */ if( zFile[0]=='|' ){ #ifdef SQLITE_OMIT_POPEN raw_printf(stderr, "Error: pipes are not supported in this OS\n"); @@ -13327,6 +14841,114 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + if( c=='p' && n>=3 && strncmp(azArg[0], "parameter", n)==0 ){ + open_db(p,0); + if( nArg<=1 ) goto parameter_syntax_error; + + /* .parameter clear + ** Clear all bind parameters by dropping the TEMP table that holds them. + */ + if( nArg==2 && strcmp(azArg[1],"clear")==0 ){ + int wrSchema = 0; + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, -1, &wrSchema); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, 0); + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp.sqlite_parameters;", + 0, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, wrSchema, 0); + }else + + /* .parameter list + ** List all bind parameters. + */ + if( nArg==2 && strcmp(azArg[1],"list")==0 ){ + sqlite3_stmt *pStmt = 0; + int rx; + int len = 0; + rx = sqlite3_prepare_v2(p->db, + "SELECT max(length(key)) " + "FROM temp.sqlite_parameters;", -1, &pStmt, 0); + if( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + len = sqlite3_column_int(pStmt, 0); + if( len>40 ) len = 40; + } + sqlite3_finalize(pStmt); + pStmt = 0; + if( len ){ + rx = sqlite3_prepare_v2(p->db, + "SELECT key, quote(value) " + "FROM temp.sqlite_parameters;", -1, &pStmt, 0); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + utf8_printf(p->out, "%-*s %s\n", len, sqlite3_column_text(pStmt,0), + sqlite3_column_text(pStmt,1)); + } + sqlite3_finalize(pStmt); + } + }else + + /* .parameter init + ** Make sure the TEMP table used to hold bind parameters exists. + ** Create it if necessary. + */ + if( nArg==2 && strcmp(azArg[1],"init")==0 ){ + bind_table_init(p); + }else + + /* .parameter set NAME VALUE + ** Set or reset a bind parameter. NAME should be the full parameter + ** name exactly as it appears in the query. (ex: $abc, @def). The + ** VALUE can be in either SQL literal notation, or if not it will be + ** understood to be a text string. + */ + if( nArg==4 && strcmp(azArg[1],"set")==0 ){ + int rx; + char *zSql; + sqlite3_stmt *pStmt; + const char *zKey = azArg[2]; + const char *zValue = azArg[3]; + bind_table_init(p); + zSql = sqlite3_mprintf( + "REPLACE INTO temp.sqlite_parameters(key,value)" + "VALUES(%Q,%s);", zKey, zValue); + if( zSql==0 ) shell_out_of_memory(); + pStmt = 0; + rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rx!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + zSql = sqlite3_mprintf( + "REPLACE INTO temp.sqlite_parameters(key,value)" + "VALUES(%Q,%Q);", zKey, zValue); + if( zSql==0 ) shell_out_of_memory(); + rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rx!=SQLITE_OK ){ + utf8_printf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); + sqlite3_finalize(pStmt); + pStmt = 0; + rc = 1; + } + } + sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + }else + + /* .parameter unset NAME + ** Remove the NAME binding from the parameter binding table, if it + ** exists. + */ + if( nArg==3 && strcmp(azArg[1],"unset")==0 ){ + char *zSql = sqlite3_mprintf( + "DELETE FROM temp.sqlite_parameters WHERE key=%Q", azArg[2]); + if( zSql==0 ) shell_out_of_memory(); + sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + }else + /* If no command name matches, show a syntax error */ + parameter_syntax_error: + showHelp(p->out, "parameter"); + }else + if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){ int i; for(i=1; i<nArg; i++){ @@ -13336,6 +14958,52 @@ static int do_meta_command(char *zLine, ShellState *p){ raw_printf(p->out, "\n"); }else +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( c=='p' && n>=3 && strncmp(azArg[0], "progress", n)==0 ){ + int i; + int nn = 0; + p->flgProgress = 0; + p->mxProgress = 0; + p->nProgress = 0; + for(i=1; i<nArg; i++){ + const char *z = azArg[i]; + if( z[0]=='-' ){ + z++; + if( z[0]=='-' ) z++; + if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){ + p->flgProgress |= SHELL_PROGRESS_QUIET; + continue; + } + if( strcmp(z,"reset")==0 ){ + p->flgProgress |= SHELL_PROGRESS_RESET; + continue; + } + if( strcmp(z,"once")==0 ){ + p->flgProgress |= SHELL_PROGRESS_ONCE; + continue; + } + if( strcmp(z,"limit")==0 ){ + if( i+1>=nArg ){ + utf8_printf(stderr, "Error: missing argument on --limit\n"); + rc = 1; + goto meta_command_exit; + }else{ + p->mxProgress = (int)integerValue(azArg[++i]); + } + continue; + } + utf8_printf(stderr, "Error: unknown option: \"%s\"\n", azArg[i]); + rc = 1; + goto meta_command_exit; + }else{ + nn = (int)integerValue(z); + } + } + open_db(p, 0); + sqlite3_progress_handler(p->db, nn, progress_handler, p); + }else +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ + if( c=='p' && strncmp(azArg[0], "prompt", n)==0 ){ if( nArg >= 2) { strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); @@ -13350,20 +15018,23 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){ - FILE *alt; + FILE *inSaved = p->in; + int savedLineno = p->lineno; if( nArg!=2 ){ raw_printf(stderr, "Usage: .read FILE\n"); rc = 1; goto meta_command_exit; } - alt = fopen(azArg[1], "rb"); - if( alt==0 ){ + p->in = fopen(azArg[1], "rb"); + if( p->in==0 ){ utf8_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ - rc = process_input(p, alt); - fclose(alt); + rc = process_input(p); + fclose(p->in); } + p->in = inSaved; + p->lineno = savedLineno; }else if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){ @@ -13387,14 +15058,14 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_open(zSrcFile, &pSrc); if( rc!=SQLITE_OK ){ utf8_printf(stderr, "Error: cannot open \"%s\"\n", zSrcFile); - sqlite3_close(pSrc); + close_db(pSrc); return 1; } open_db(p, 0); pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main"); if( pBackup==0 ){ utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); - sqlite3_close(pSrc); + close_db(pSrc); return 1; } while( (rc = sqlite3_backup_step(pBackup,100))==SQLITE_OK @@ -13414,13 +15085,12 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); rc = 1; } - sqlite3_close(pSrc); + close_db(pSrc); }else - if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ - p->scanstatsOn = booleanValue(azArg[1]); + p->scanstatsOn = (u8)booleanValue(azArg[1]); #ifndef SQLITE_ENABLE_STMT_SCANSTATUS raw_printf(stderr, "Warning: .scanstats not available in this build.\n"); #endif @@ -13459,8 +15129,8 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( zName!=0 ){ - int isMaster = sqlite3_strlike(zName, "sqlite_master", 0)==0; - if( isMaster || sqlite3_strlike(zName,"sqlite_temp_master",0)==0 ){ + int isMaster = sqlite3_strlike(zName, "sqlite_master", '\\')==0; + if( isMaster || sqlite3_strlike(zName,"sqlite_temp_master", '\\')==0 ){ char *new_argv[2], *new_colv[2]; new_argv[0] = sqlite3_mprintf( "CREATE TABLE %s (\n" @@ -13520,13 +15190,18 @@ static int do_meta_command(char *zLine, ShellState *p){ appendText(&sSelect, ") WHERE ", 0); if( zName ){ char *zQarg = sqlite3_mprintf("%Q", zName); + int bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || + strchr(zName, '[') != 0; if( strchr(zName, '.') ){ appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0); }else{ appendText(&sSelect, "lower(tbl_name)", 0); } - appendText(&sSelect, strchr(zName, '*') ? " GLOB " : " LIKE ", 0); + appendText(&sSelect, bGlob ? " GLOB " : " LIKE ", 0); appendText(&sSelect, zQarg, 0); + if( !bGlob ){ + appendText(&sSelect, " ESCAPE '\\' ", 0); + } appendText(&sSelect, " AND ", 0); sqlite3_free(zQarg); } @@ -13749,7 +15424,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else /* If no command name matches, show a syntax error */ session_syntax_error: - session_help(p); + showHelp(p->out, "session"); }else #endif @@ -13930,7 +15605,7 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); raw_printf(stderr, "Should be one of: --schema" - " --sha3-224 --sha3-255 --sha3-384 --sha3-512\n"); + " --sha3-224 --sha3-256 --sha3-384 --sha3-512\n"); rc = 1; goto meta_command_exit; } @@ -13941,7 +15616,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ zLike = z; bSeparate = 1; - if( sqlite3_strlike("sqlite_%", zLike, 0)==0 ) bSchema = 1; + if( sqlite3_strlike("sqlite\\_%", zLike, '\\')==0 ) bSchema = 1; } } if( bSchema ){ @@ -14008,11 +15683,12 @@ static int do_meta_command(char *zLine, ShellState *p){ if( bDebug ){ utf8_printf(p->out, "%s\n", zSql); }else{ - shell_exec(p->db, zSql, shell_callback, p, 0); + shell_exec(p, zSql, 0); } sqlite3_free(zSql); }else +#ifndef SQLITE_NOHAVE_SYSTEM if( c=='s' && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) ){ @@ -14032,6 +15708,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zCmd); if( x ) raw_printf(stderr, "System command returns %d\n", x); }else +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ static const char *azBool[] = { "off", "on", "trigger", "full"}; @@ -14071,7 +15748,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){ if( nArg==2 ){ - p->statsOn = booleanValue(azArg[1]); + p->statsOn = (u8)booleanValue(azArg[1]); }else if( nArg==1 ){ display_stats(p->db, p, 0); }else{ @@ -14092,7 +15769,10 @@ static int do_meta_command(char *zLine, ShellState *p){ initText(&s); open_db(p, 0); rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); - if( rc ) return shellDatabaseError(p->db); + if( rc ){ + sqlite3_finalize(pStmt); + return shellDatabaseError(p->db); + } if( nArg>2 && c=='i' ){ /* It is an historical accident that the .indexes command shows an error @@ -14100,6 +15780,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** command does not. */ raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n"); rc = 1; + sqlite3_finalize(pStmt); goto meta_command_exit; } for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){ @@ -14144,18 +15825,12 @@ static int do_meta_command(char *zLine, ShellState *p){ char **azNew; int n2 = nAlloc*2 + 10; azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2); - if( azNew==0 ){ - rc = shellNomemError(); - break; - } + if( azNew==0 ) shell_out_of_memory(); nAlloc = n2; azResult = azNew; } azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); - if( 0==azResult[nRow] ){ - rc = shellNomemError(); - break; - } + if( 0==azResult[nRow] ) shell_out_of_memory(); nRow++; } if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ @@ -14216,9 +15891,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { "byteorder", SQLITE_TESTCTRL_BYTEORDER, "" }, /*{ "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, "" }, */ { "imposter", SQLITE_TESTCTRL_IMPOSTER, "SCHEMA ON/OFF ROOTPAGE"}, -#ifdef SQLITE_N_KEYWORD - { "iskeyword", SQLITE_TESTCTRL_ISKEYWORD, "IDENTIFIER" }, -#endif + { "internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, "BOOLEAN" }, { "localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN" }, { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN" }, { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK" }, @@ -14313,6 +15986,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* sqlite3_test_control(int, int) */ case SQLITE_TESTCTRL_ASSERT: case SQLITE_TESTCTRL_ALWAYS: + case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: if( nArg==3 ){ int opt = booleanValue(azArg[2]); rc2 = sqlite3_test_control(testctrl, opt); @@ -14330,17 +16004,6 @@ static int do_meta_command(char *zLine, ShellState *p){ } break; - /* sqlite3_test_control(int, char *) */ -#ifdef SQLITE_N_KEYWORD - case SQLITE_TESTCTRL_ISKEYWORD: - if( nArg==3 ){ - const char *opt = azArg[2]; - rc2 = sqlite3_test_control(testctrl, opt); - isOk = 1; - } - break; -#endif - case SQLITE_TESTCTRL_IMPOSTER: if( nArg==5 ){ rc2 = sqlite3_test_control(testctrl, p->db, @@ -14389,23 +16052,55 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else +#ifndef SQLITE_OMIT_TRACE if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ + int mType = 0; + int jj; open_db(p, 0); - if( nArg!=2 ){ - raw_printf(stderr, "Usage: .trace FILE|off\n"); - rc = 1; - goto meta_command_exit; + for(jj=1; jj<nArg; jj++){ + const char *z = azArg[jj]; + if( z[0]=='-' ){ + if( optionMatch(z, "expanded") ){ + p->eTraceType = SHELL_TRACE_EXPANDED; + } +#ifdef SQLITE_ENABLE_NORMALIZE + else if( optionMatch(z, "normalized") ){ + p->eTraceType = SHELL_TRACE_NORMALIZED; + } +#endif + else if( optionMatch(z, "plain") ){ + p->eTraceType = SHELL_TRACE_PLAIN; + } + else if( optionMatch(z, "profile") ){ + mType |= SQLITE_TRACE_PROFILE; + } + else if( optionMatch(z, "row") ){ + mType |= SQLITE_TRACE_ROW; + } + else if( optionMatch(z, "stmt") ){ + mType |= SQLITE_TRACE_STMT; + } + else if( optionMatch(z, "close") ){ + mType |= SQLITE_TRACE_CLOSE; + } + else { + raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z); + rc = 1; + goto meta_command_exit; + } + }else{ + output_file_close(p->traceOut); + p->traceOut = output_file_open(azArg[1], 0); + } } - output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) if( p->traceOut==0 ){ sqlite3_trace_v2(p->db, 0, 0, 0); }else{ - sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut); + if( mType==0 ) mType = SQLITE_TRACE_STMT; + sqlite3_trace_v2(p->db, mType, sql_trace_callback, p); } -#endif }else +#endif /* !defined(SQLITE_OMIT_TRACE) */ #if SQLITE_USER_AUTHENTICATION if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ @@ -14612,6 +16307,16 @@ static int line_is_command_terminator(const char *zLine){ } /* +** We need a default sqlite3_complete() implementation to use in case +** the shell is compiled with SQLITE_OMIT_COMPLETE. The default assumes +** any arbitrary text is a complete SQL statement. This is not very +** user-friendly, but it does seem to work. +*/ +#ifdef SQLITE_OMIT_COMPLETE +#define sqlite3_complete(x) 1 +#endif + +/* ** Return true if zSql is a complete SQL statement. Return false if it ** ends in the middle of a string literal or C-style comment. */ @@ -14626,7 +16331,7 @@ static int line_is_complete(char *zSql, int nSql){ } /* -** Run a single line of SQL +** Run a single line of SQL. Return the number of errors. */ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ int rc; @@ -14634,8 +16339,9 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ open_db(p, 0); if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql); + if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; BEGIN_TIMER; - rc = shell_exec(p->db, zSql, shell_callback, p, &zErrMsg); + rc = shell_exec(p, zSql, &zErrMsg); END_TIMER; if( rc || zErrMsg ){ char zPrefix[100]; @@ -14670,7 +16376,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ ** ** Return the number of errors. */ -static int process_input(ShellState *p, FILE *in){ +static int process_input(ShellState *p){ char *zLine = 0; /* A single input line */ char *zSql = 0; /* Accumulated SQL text */ int nLine; /* Length of current line */ @@ -14679,33 +16385,35 @@ static int process_input(ShellState *p, FILE *in){ int nSqlPrior = 0; /* Bytes of zSql[] used by prior line */ int rc; /* Error code */ int errCnt = 0; /* Number of errors seen */ - int lineno = 0; /* Current line number */ int startline = 0; /* Line number for start of current input */ - while( errCnt==0 || !bail_on_error || (in==0 && stdin_is_interactive) ){ + p->lineno = 0; + while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ fflush(p->out); - zLine = one_input_line(in, zLine, nSql>0); + zLine = one_input_line(p->in, zLine, nSql>0); if( zLine==0 ){ /* End of input */ - if( in==0 && stdin_is_interactive ) printf("\n"); + if( p->in==0 && stdin_is_interactive ) printf("\n"); break; } if( seenInterrupt ){ - if( in!=0 ) break; + if( p->in!=0 ) break; seenInterrupt = 0; } - lineno++; + p->lineno++; if( nSql==0 && _all_whitespace(zLine) ){ if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); continue; } - if( zLine && zLine[0]=='.' && nSql==0 ){ + if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){ if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); - rc = do_meta_command(zLine, p); - if( rc==2 ){ /* exit requested */ - break; - }else if( rc ){ - errCnt++; + if( zLine[0]=='.' ){ + rc = do_meta_command(zLine, p); + if( rc==2 ){ /* exit requested */ + break; + }else if( rc ){ + errCnt++; + } } continue; } @@ -14716,10 +16424,7 @@ static int process_input(ShellState *p, FILE *in){ if( nSql+nLine+2>=nAlloc ){ nAlloc = nSql+nLine+100; zSql = realloc(zSql, nAlloc); - if( zSql==0 ){ - raw_printf(stderr, "Error: out of memory\n"); - exit(1); - } + if( zSql==0 ) shell_out_of_memory(); } nSqlPrior = nSql; if( nSql==0 ){ @@ -14727,7 +16432,7 @@ static int process_input(ShellState *p, FILE *in){ for(i=0; zLine[i] && IsSpace(zLine[i]); i++){} assert( nAlloc>0 && zSql!=0 ); memcpy(zSql, zLine+i, nLine+1-i); - startline = lineno; + startline = p->lineno; nSql = nLine-i; }else{ zSql[nSql++] = '\n'; @@ -14736,7 +16441,7 @@ static int process_input(ShellState *p, FILE *in){ } if( nSql && line_contains_semicolon(&zSql[nSqlPrior], nSql-nSqlPrior) && sqlite3_complete(zSql) ){ - errCnt += runOneSqlLine(p, zSql, in, startline); + errCnt += runOneSqlLine(p, zSql, p->in, startline); nSql = 0; if( p->outCount ){ output_reset(p); @@ -14750,7 +16455,7 @@ static int process_input(ShellState *p, FILE *in){ } } if( nSql && !_all_whitespace(zSql) ){ - runOneSqlLine(p, zSql, in, startline); + errCnt += runOneSqlLine(p, zSql, p->in, startline); } free(zSql); free(zLine); @@ -14839,7 +16544,8 @@ static void process_sqliterc( char *home_dir = NULL; const char *sqliterc = sqliterc_override; char *zBuf = 0; - FILE *in = NULL; + FILE *inSaved = p->in; + int savedLineno = p->lineno; if (sqliterc == NULL) { home_dir = find_home_dir(0); @@ -14848,18 +16554,19 @@ static void process_sqliterc( " cannot read ~/.sqliterc\n"); return; } - sqlite3_initialize(); zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir); sqliterc = zBuf; } - in = fopen(sqliterc,"rb"); - if( in ){ + p->in = fopen(sqliterc,"rb"); + if( p->in ){ if( stdin_is_interactive ){ utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc); } - process_input(p,in); - fclose(in); + process_input(p); + fclose(p->in); } + p->in = inSaved; + p->lineno = savedLineno; sqlite3_free(zBuf); } @@ -14867,12 +16574,19 @@ static void process_sqliterc( ** Show available command line options */ static const char zOptions[] = +#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) + " -A ARGS... run \".archive ARGS\" and exit\n" +#endif + " -append append the database to the end of the file\n" " -ascii set output mode to 'ascii'\n" " -bail stop after hitting an error\n" " -batch force batch I/O\n" " -column set output mode to 'column'\n" " -cmd COMMAND run \"COMMAND\" before reading stdin\n" " -csv set output mode to 'csv'\n" +#if defined(SQLITE_ENABLE_DESERIALIZE) + " -deserialize open the database using sqlite3_deserialize()\n" +#endif " -echo print commands before execution\n" " -init FILENAME read/process named file\n" " -[no]header turn headers on or off\n" @@ -14885,6 +16599,10 @@ static const char zOptions[] = " -line set output mode to 'line'\n" " -list set output mode to 'list'\n" " -lookaside SIZE N use N entries of SZ bytes for lookaside memory\n" +#if defined(SQLITE_ENABLE_DESERIALIZE) + " -maxsize N maximum size for a --deserialize database\n" +#endif + " -memtrace trace all memory allocations and deallocations\n" " -mmap N default mmap size set to N\n" #ifdef SQLITE_ENABLE_MULTIPLEX " -multiplex enable the multiplexor VFS\n" @@ -14893,13 +16611,20 @@ static const char zOptions[] = " -nullvalue TEXT set text string for NULL values. Default ''\n" " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" " -quote set output mode to 'quote'\n" + " -readonly open the database read-only\n" " -separator SEP set output column separator. Default: '|'\n" +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + " -sorterref SIZE sorter references threshold size\n" +#endif " -stats print memory stats before each finalize\n" " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" #ifdef SQLITE_ENABLE_VFSTRACE " -vfstrace enable tracing of all VFS calls\n" #endif +#ifdef SQLITE_HAVE_ZLIB + " -zip open the file as a ZIP Archive\n" +#endif ; static void usage(int showDetail){ utf8_printf(stderr, @@ -14915,6 +16640,17 @@ static void usage(int showDetail){ } /* +** Internal check: Verify that the SQLite is uninitialized. Print a +** error message if it is initialized. +*/ +static void verify_uninitialized(void){ + if( sqlite3_config(-1)==SQLITE_MISUSE ){ + utf8_printf(stdout, "WARNING: attempt to configure SQLite after" + " initialization.\n"); + } +} + +/* ** Initialize the state information in data */ static void main_init(ShellState *data) { @@ -14925,6 +16661,7 @@ static void main_init(ShellState *data) { memcpy(data->rowSeparator,SEP_Row, 2); data->showHeader = 0; data->shellFlgs = SHFLG_Lookaside; + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); @@ -14988,12 +16725,34 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ int readStdin = 1; int nCmd = 0; char **azCmd = 0; + const char *zVfs = 0; /* Value of -vfs command-line option */ +#if !SQLITE_SHELL_IS_UTF8 + char **argvToFree = 0; + int argcToFree = 0; +#endif setBinaryMode(stdin, 0); setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ stdin_is_interactive = isatty(0); stdout_is_console = isatty(1); +#if !defined(_WIN32_WCE) + if( getenv("SQLITE_DEBUG_BREAK") ){ + if( isatty(0) && isatty(2) ){ + fprintf(stderr, + "attach debugger to process %d and press any key to continue.\n", + GETPID()); + fgetc(stdin); + }else{ +#if defined(_WIN32) || defined(WIN32) + DebugBreak(); +#elif defined(SIGTRAP) + raise(SIGTRAP); +#endif + } + } +#endif + #if USE_SYSTEM_SQLITE+0!=1 if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", @@ -15002,21 +16761,33 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #endif main_init(&data); + + /* On Windows, we must translate command-line arguments into UTF-8. + ** The SQLite memory allocator subsystem has to be enabled in order to + ** do this. But we want to run an sqlite3_shutdown() afterwards so that + ** subsequent sqlite3_config() calls will work. So copy all results into + ** memory that does not come from the SQLite memory allocator. + */ #if !SQLITE_SHELL_IS_UTF8 sqlite3_initialize(); - argv = sqlite3_malloc64(sizeof(argv[0])*argc); - if( argv==0 ){ - raw_printf(stderr, "out of memory\n"); - exit(1); - } + argvToFree = malloc(sizeof(argv[0])*argc*2); + argcToFree = argc; + argv = argvToFree + argc; + if( argv==0 ) shell_out_of_memory(); for(i=0; i<argc; i++){ - argv[i] = sqlite3_win32_unicode_to_utf8(wargv[i]); - if( argv[i]==0 ){ - raw_printf(stderr, "out of memory\n"); - exit(1); - } + char *z = sqlite3_win32_unicode_to_utf8(wargv[i]); + int n; + if( z==0 ) shell_out_of_memory(); + n = (int)strlen(z); + argv[i] = malloc( n+1 ); + if( argv[i]==0 ) shell_out_of_memory(); + memcpy(argv[i], z, n+1); + argvToFree[i] = argv[i]; + sqlite3_free(z); } + sqlite3_shutdown(); #endif + assert( argc>=1 && argv && argv[0] ); Argv0 = argv[0]; @@ -15046,6 +16817,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** the size of the alternative malloc heap, ** and the first command to execute. */ + verify_uninitialized(); for(i=1; i<argc; i++){ char *z; z = argv[i]; @@ -15058,10 +16830,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ readStdin = 0; nCmd++; azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd); - if( azCmd==0 ){ - raw_printf(stderr, "out of memory\n"); - exit(1); - } + if( azCmd==0 ) shell_out_of_memory(); azCmd[nCmd-1] = z; } } @@ -15128,22 +16897,65 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( strcmp(z,"-mmap")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + }else if( strcmp(z,"-sorterref")==0 ){ + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz); +#endif }else if( strcmp(z,"-vfs")==0 ){ - sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i)); - if( pVfs ){ - sqlite3_vfs_register(pVfs, 1); - }else{ - utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]); - exit(1); - } -#ifdef SQLITE_HAVE_ZIP + zVfs = cmdline_option_value(argc, argv, ++i); +#ifdef SQLITE_HAVE_ZLIB }else if( strcmp(z,"-zip")==0 ){ data.openMode = SHELL_OPEN_ZIPFILE; #endif }else if( strcmp(z,"-append")==0 ){ data.openMode = SHELL_OPEN_APPENDVFS; +#ifdef SQLITE_ENABLE_DESERIALIZE + }else if( strcmp(z,"-deserialize")==0 ){ + data.openMode = SHELL_OPEN_DESERIALIZE; + }else if( strcmp(z,"-maxsize")==0 && i+1<argc ){ + data.szMax = integerValue(argv[++i]); +#endif + }else if( strcmp(z,"-readonly")==0 ){ + data.openMode = SHELL_OPEN_READONLY; +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) + }else if( strncmp(z, "-A",2)==0 ){ + /* All remaining command-line arguments are passed to the ".archive" + ** command, so ignore them */ + break; +#endif + }else if( strcmp(z, "-memtrace")==0 ){ + sqlite3MemTraceActivate(stderr); } } + verify_uninitialized(); + + +#ifdef SQLITE_SHELL_INIT_PROC + { + /* If the SQLITE_SHELL_INIT_PROC macro is defined, then it is the name + ** of a C-function that will perform initialization actions on SQLite that + ** occur just before or after sqlite3_initialize(). Use this compile-time + ** option to embed this shell program in larger applications. */ + extern void SQLITE_SHELL_INIT_PROC(void); + SQLITE_SHELL_INIT_PROC(); + } +#else + /* All the sqlite3_config() calls have now been made. So it is safe + ** to call sqlite3_initialize() and process any command line -vfs option. */ + sqlite3_initialize(); +#endif + + if( zVfs ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs); + if( pVfs ){ + sqlite3_vfs_register(pVfs, 1); + }else{ + utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]); + exit(1); + } + } + if( data.zDbFilename==0 ){ #ifndef SQLITE_OMIT_MEMORYDB data.zDbFilename = ":memory:"; @@ -15195,12 +17007,20 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( strcmp(z,"-csv")==0 ){ data.mode = MODE_Csv; memcpy(data.colSeparator,",",2); -#ifdef SQLITE_HAVE_ZIP +#ifdef SQLITE_HAVE_ZLIB }else if( strcmp(z,"-zip")==0 ){ data.openMode = SHELL_OPEN_ZIPFILE; #endif }else if( strcmp(z,"-append")==0 ){ data.openMode = SHELL_OPEN_APPENDVFS; +#ifdef SQLITE_ENABLE_DESERIALIZE + }else if( strcmp(z,"-deserialize")==0 ){ + data.openMode = SHELL_OPEN_DESERIALIZE; + }else if( strcmp(z,"-maxsize")==0 && i+1<argc ){ + data.szMax = integerValue(argv[++i]); +#endif + }else if( strcmp(z,"-readonly")==0 ){ + data.openMode = SHELL_OPEN_READONLY; }else if( strcmp(z,"-ascii")==0 ){ data.mode = MODE_Ascii; sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, @@ -15254,6 +17074,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ i+=2; }else if( strcmp(z,"-mmap")==0 ){ i++; + }else if( strcmp(z,"-memtrace")==0 ){ + i++; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + }else if( strcmp(z,"-sorterref")==0 ){ + i++; +#endif }else if( strcmp(z,"-vfs")==0 ){ i++; #ifdef SQLITE_ENABLE_VFSTRACE @@ -15278,7 +17104,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( rc && bail_on_error ) return rc==2 ? 0 : rc; }else{ open_db(&data, 0); - rc = shell_exec(data.db, z, shell_callback, &data, &zErrMsg); + rc = shell_exec(&data, z, &zErrMsg); if( zErrMsg!=0 ){ utf8_printf(stderr,"Error: %s\n", zErrMsg); if( bail_on_error ) return rc!=0 ? rc : 1; @@ -15287,6 +17113,23 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( bail_on_error ) return rc; } } +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) + }else if( strncmp(z, "-A", 2)==0 ){ + if( nCmd>0 ){ + utf8_printf(stderr, "Error: cannot mix regular SQL or dot-commands" + " with \"%s\"\n", z); + return 1; + } + open_db(&data, OPEN_DB_ZIPFILE); + if( z[2] ){ + argv[i] = &z[2]; + arDotCommand(&data, 1, argv+(i-1), argc-(i-1)); + }else{ + arDotCommand(&data, 1, argv+i, argc-i); + } + readStdin = 0; + break; +#endif }else{ utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); raw_printf(stderr,"Use -help for a list of options.\n"); @@ -15306,7 +17149,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( rc ) return rc==2 ? 0 : rc; }else{ open_db(&data, 0); - rc = shell_exec(data.db, azCmd[i], shell_callback, &data, &zErrMsg); + rc = shell_exec(&data, azCmd[i], &zErrMsg); if( zErrMsg!=0 ){ utf8_printf(stderr,"Error: %s\n", zErrMsg); return rc!=0 ? rc : 1; @@ -15322,7 +17165,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ */ if( stdin_is_interactive ){ char *zHome; - char *zHistory = 0; + char *zHistory; int nHistory; printf( "SQLite version %s %.19s\n" /*extra-version-info*/ @@ -15335,8 +17178,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ printf(".\nUse \".open FILENAME\" to reopen on a " "persistent database.\n"); } - zHome = find_home_dir(0); - if( zHome ){ + zHistory = getenv("SQLITE_HISTORY"); + if( zHistory ){ + zHistory = strdup(zHistory); + }else if( (zHome = find_home_dir(0))!=0 ){ nHistory = strlen30(zHome) + 20; if( (zHistory = malloc(nHistory))!=0 ){ sqlite3_snprintf(nHistory, zHistory,"%s/.sqlite_history", zHome); @@ -15348,20 +17193,22 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #elif HAVE_LINENOISE linenoiseSetCompletionCallback(linenoise_completion); #endif - rc = process_input(&data, 0); + data.in = 0; + rc = process_input(&data); if( zHistory ){ shell_stifle_history(2000); shell_write_history(zHistory); free(zHistory); } }else{ - rc = process_input(&data, stdin); + data.in = stdin; + rc = process_input(&data); } } set_table_name(&data, 0); if( data.db ){ session_close_all(&data); - sqlite3_close(data.db); + close_db(data.db); } sqlite3_free(data.zFreeOnClose); find_home_dir(1); @@ -15369,8 +17216,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.doXdgOpen = 0; clearTempFile(&data); #if !SQLITE_SHELL_IS_UTF8 - for(i=0; i<argc; i++) sqlite3_free(argv[i]); - sqlite3_free(argv); + for(i=0; i<argcToFree; i++) free(argvToFree[i]); + free(argvToFree); #endif + /* Clear the global data structure so that valgrind will detect memory + ** leaks */ + memset(&data, 0, sizeof(data)); return rc; } |