Test cases for sqlite3_normalize().

FossilOrigin-Name: 658f42257d56a3562dfa8e55023c6e497f55c565cd751d942de3a9e0cb2fa708

Makefile.in

@@ -440,6 +440,7 @@ TESTSRC += \
   $(TOP)/ext/misc/ieee754.c \
   $(TOP)/ext/misc/mmapwarm.c \
   $(TOP)/ext/misc/nextchar.c \
+  $(TOP)/ext/misc/normalize.c \
   $(TOP)/ext/misc/percentile.c \
   $(TOP)/ext/misc/regexp.c \
   $(TOP)/ext/misc/remember.c \
@@ -447,7 +448,8 @@ TESTSRC += \
   $(TOP)/ext/misc/spellfix.c \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/unionvtab.c \
-  $(TOP)/ext/misc/wholenumber.c
+  $(TOP)/ext/misc/wholenumber.c \
+  $(TOP)/ext/misc/zipfile.c
 
 # Source code to the library files needed by the test fixture
 #
@@ -558,7 +560,8 @@ TESTPROGS = \
   sqlite3$(TEXE) \
   sqlite3_analyzer$(TEXE) \
   sqldiff$(TEXE) \
-  dbhash$(TEXE)
+  dbhash$(TEXE) \
+  sqltclsh$(TEXE)
 
 # Databases containing fuzzer test cases
 #
@@ -992,6 +995,7 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 # Source files that go into making shell.c
 SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
+        $(TOP)/ext/misc/appendvfs.c \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
 	$(TOP)/ext/misc/completion.c \
@@ -1199,6 +1203,12 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in	
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
+
+sqltclsh$(TEXE): sqltclsh.c
+	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
+
 sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
 	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
 

Makefile.msc

@@ -1500,6 +1500,7 @@ TESTEXT = \
   $(TOP)\ext\misc\ieee754.c \
   $(TOP)\ext\misc\mmapwarm.c \
   $(TOP)\ext\misc\nextchar.c \
+  $(TOP)\ext\misc\normalize.c \
   $(TOP)\ext\misc\percentile.c \
   $(TOP)\ext\misc\regexp.c \
   $(TOP)\ext\misc\remember.c \
@@ -1586,7 +1587,8 @@ TESTPROGS = \
   sqlite3_analyzer.exe \
   sqlite3_checker.exe \
   sqldiff.exe \
-  dbhash.exe
+  dbhash.exe \
+  sqltclsh.exe
 
 # Databases containing fuzzer test cases
 #
@@ -2082,6 +2084,7 @@ keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 # Source files that go into making shell.c
 SHELL_SRC = \
 	$(TOP)\src\shell.c.in \
+        $(TOP)\ext\misc\appendvfs.c \
 	$(TOP)\ext\misc\shathree.c \
 	$(TOP)\ext\misc\fileio.c \
 	$(TOP)\ext\misc\completion.c \
@@ -2334,6 +2337,13 @@ sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
+sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c
+
+sqltclsh.exe: sqltclsh.c
+	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqltclsh.c \
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
+
 sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
 
@@ -2352,7 +2362,7 @@ sqlite3_checker.c:	$(CHECKER_DEPS)
 
 sqlite3_checker.exe:	sqlite3_checker.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \
-		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
 dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
@@ -2453,6 +2463,7 @@ clean:
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
 	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
+	del /Q sqltclsh.exe 2>NUL
 	del /Q fts5.* fts5parse.* 2>NUL
 	del /Q lsm.h lsm1.c 2>NUL
 # <</mark>>

autoconf/Makefile.msc

@@ -930,13 +930,22 @@ LIBRESOBJS =
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_INTROSPECTION_PRAGMAS
 !ENDIF
 
 
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-all:	dll shell
+core:	dll shell
+
+# Targets that require the Tcl library.
+#
+tcl:	$(ALL_TCL_TARGETS)
+
+# This Makefile target builds all of the standard binaries.
+#
+all:	core tcl
 
 # Dynamic link library section.
 #

ext/lsm1/lsm-test/lsmtest.h

@@ -121,6 +121,7 @@ int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int,
 */
 int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
 int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
+int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);

ext/lsm1/lsm-test/lsmtest_tdb.c

@@ -721,6 +721,7 @@ static struct Lib {
   { "sqlite3",      "testdb.sqlite",    sql_open },
   { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
   { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
+  { "lsm_lomem2",   "testdb.lsm_lomem2", test_lsm_lomem2_open },
 #ifdef HAVE_ZLIB
   { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
 #endif

ext/lsm1/lsm-test/lsmtest_tdb3.c

@@ -1033,6 +1033,19 @@ int test_lsm_lomem_open(
   return testLsmOpen(zCfg, zFilename, bClear, ppDb);
 }
 
+int test_lsm_lomem2_open(
+  const char *zSpec, 
+  const char *zFilename, 
+  int bClear, 
+  TestDb **ppDb
+){
+    /* "max_freelist=4 autocheckpoint=32" */
+  const char *zCfg = 
+    "page_size=512 block_size=64 autoflush=0 mmap=0 "
+  ;
+  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
+}
+
 int test_lsm_zip_open(
   const char *zSpec, 
   const char *zFilename, 

ext/lsm1/lsm_main.c

@@ -683,7 +683,7 @@ static int doWriteOp(
     int nDiff;
 
     if( nQuant>pDb->nTreeLimit ){
-      nQuant = pDb->nTreeLimit;
+      nQuant = LSM_MAX(pDb->nTreeLimit, pgsz);
     }
 
     nBefore = lsmTreeSize(pDb);

ext/lsm1/lsm_sorted.c

@@ -5252,16 +5252,15 @@ static int doLsmSingleWork(
   /* If the in-memory part of the free-list is too large, write a new 
   ** top-level containing just the in-memory free-list entries to disk. */
   if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){
-    int nPg = 0;
     while( rc==LSM_OK && lsmDatabaseFull(pDb) ){
+      int nPg = 0;
       rc = sortedWork(pDb, 16, nMerge, 1, &nPg);
       nRem -= nPg;
     }
     if( rc==LSM_OK ){
       rc = sortedNewFreelistOnly(pDb);
     }
-    nRem -= nPg;
-    if( nPg ) bDirty = 1;
+    bDirty = 1;
   }
 
   if( rc==LSM_OK ){

ext/misc/appendvfs.c

@@ -0,0 +1,563 @@
+/*
+** 2017-10-20
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file implements a VFS shim that allows an SQLite database to be
+** appended onto the end of some other file, such as an executable.
+**
+** A special record must appear at the end of the file that identifies the
+** file as an appended database and provides an offset to page 1.  For
+** best performance page 1 should be located at a disk page boundary, though
+** that is not required.
+**
+** When opening a database using this VFS, the connection might treat
+** the file as an ordinary SQLite database, or it might treat is as a
+** database appended onto some other file.  Here are the rules:
+**
+**  (1)  When opening a new empty file, that file is treated as an ordinary
+**       database.
+**
+**  (2)  When opening a file that begins with the standard SQLite prefix
+**       string "SQLite format 3", that file is treated as an ordinary
+**       database.
+**
+**  (3)  When opening a file that ends with the appendvfs trailer string
+**       "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
+**       database.
+**
+**  (4)  If none of the above apply and the SQLITE_OPEN_CREATE flag is
+**       set, then a new database is appended to the already existing file.
+**
+**  (5)  Otherwise, SQLITE_CANTOPEN is returned.
+**
+** To avoid unnecessary complications with the PENDING_BYTE, the size of
+** the file containing the database is limited to 1GB.  This VFS will refuse
+** to read or write past the 1GB mark.  This restriction might be lifted in
+** future versions.  For now, if you need a large database, then keep the
+** database in a separate file.
+**
+** If the file being opened is not an appended database, then this shim is
+** a pass-through into the default underlying VFS.
+**/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+
+/* The append mark at the end of the database is:
+**
+**     Start-Of-SQLite3-NNNNNNNN
+**     123456789 123456789 12345
+**
+** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
+** the offset to page 1.
+*/
+#define APND_MARK_PREFIX     "Start-Of-SQLite3-"
+#define APND_MARK_PREFIX_SZ  17
+#define APND_MARK_SIZE       25
+
+/*
+** Maximum size of the combined prefix + database + append-mark.  This
+** must be less than 0x40000000 to avoid locking issues on Windows.
+*/
+#define APND_MAX_SIZE  (65536*15259)
+
+/*
+** Forward declaration of objects used by this utility
+*/
+typedef struct sqlite3_vfs ApndVfs;
+typedef struct ApndFile ApndFile;
+
+/* Access to a lower-level VFS that (might) implement dynamic loading,
+** access to randomness, etc.
+*/
+#define ORIGVFS(p)  ((sqlite3_vfs*)((p)->pAppData))
+#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
+
+/* An open file */
+struct ApndFile {
+  sqlite3_file base;              /* IO methods */
+  sqlite3_int64 iPgOne;           /* File offset to page 1 */
+  sqlite3_int64 iMark;            /* Start of the append-mark */
+};
+
+/*
+** Methods for ApndFile
+*/
+static int apndClose(sqlite3_file*);
+static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
+static int apndTruncate(sqlite3_file*, sqlite3_int64 size);
+static int apndSync(sqlite3_file*, int flags);
+static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int apndLock(sqlite3_file*, int);
+static int apndUnlock(sqlite3_file*, int);
+static int apndCheckReservedLock(sqlite3_file*, int *pResOut);
+static int apndFileControl(sqlite3_file*, int op, void *pArg);
+static int apndSectorSize(sqlite3_file*);
+static int apndDeviceCharacteristics(sqlite3_file*);
+static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
+static int apndShmLock(sqlite3_file*, int offset, int n, int flags);
+static void apndShmBarrier(sqlite3_file*);
+static int apndShmUnmap(sqlite3_file*, int deleteFlag);
+static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
+static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
+
+/*
+** Methods for ApndVfs
+*/
+static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
+static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir);
+static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *);
+static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
+static void *apndDlOpen(sqlite3_vfs*, const char *zFilename);
+static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
+static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
+static void apndDlClose(sqlite3_vfs*, void*);
+static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut);
+static int apndSleep(sqlite3_vfs*, int microseconds);
+static int apndCurrentTime(sqlite3_vfs*, double*);
+static int apndGetLastError(sqlite3_vfs*, int, char *);
+static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr);
+static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z);
+static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName);
+
+static sqlite3_vfs apnd_vfs = {
+  3,                            /* iVersion (set when registered) */
+  0,                            /* szOsFile (set when registered) */
+  1024,                         /* mxPathname */
+  0,                            /* pNext */
+  "apndvfs",                    /* zName */
+  0,                            /* pAppData (set when registered) */ 
+  apndOpen,                     /* xOpen */
+  apndDelete,                   /* xDelete */
+  apndAccess,                   /* xAccess */
+  apndFullPathname,             /* xFullPathname */
+  apndDlOpen,                   /* xDlOpen */
+  apndDlError,                  /* xDlError */
+  apndDlSym,                    /* xDlSym */
+  apndDlClose,                  /* xDlClose */
+  apndRandomness,               /* xRandomness */
+  apndSleep,                    /* xSleep */
+  apndCurrentTime,              /* xCurrentTime */
+  apndGetLastError,             /* xGetLastError */
+  apndCurrentTimeInt64,         /* xCurrentTimeInt64 */
+  apndSetSystemCall,            /* xSetSystemCall */
+  apndGetSystemCall,            /* xGetSystemCall */
+  apndNextSystemCall            /* xNextSystemCall */
+};
+
+static const sqlite3_io_methods apnd_io_methods = {
+  3,                              /* iVersion */
+  apndClose,                      /* xClose */
+  apndRead,                       /* xRead */
+  apndWrite,                      /* xWrite */
+  apndTruncate,                   /* xTruncate */
+  apndSync,                       /* xSync */
+  apndFileSize,                   /* xFileSize */
+  apndLock,                       /* xLock */
+  apndUnlock,                     /* xUnlock */
+  apndCheckReservedLock,          /* xCheckReservedLock */
+  apndFileControl,                /* xFileControl */
+  apndSectorSize,                 /* xSectorSize */
+  apndDeviceCharacteristics,      /* xDeviceCharacteristics */
+  apndShmMap,                     /* xShmMap */
+  apndShmLock,                    /* xShmLock */
+  apndShmBarrier,                 /* xShmBarrier */
+  apndShmUnmap,                   /* xShmUnmap */
+  apndFetch,                      /* xFetch */
+  apndUnfetch                     /* xUnfetch */
+};
+
+
+
+/*
+** Close an apnd-file.
+*/
+static int apndClose(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xClose(pFile);
+}
+
+/*
+** Read data from an apnd-file.
+*/
+static int apndRead(
+  sqlite3_file *pFile, 
+  void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
+}
+
+/*
+** Add the append-mark onto the end of the file.
+*/
+static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
+  int i;
+  unsigned char a[APND_MARK_SIZE];
+  memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
+  for(i=0; i<8; i++){
+    a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
+  }
+  return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark);
+}
+
+/*
+** Write data to an apnd-file.
+*/
+static int apndWrite(
+  sqlite3_file *pFile,
+  const void *zBuf,
+  int iAmt,
+  sqlite_int64 iOfst
+){
+  int rc;
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL;
+  rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
+  if( rc==SQLITE_OK &&  iOfst + iAmt + p->iPgOne > p->iMark ){
+    sqlite3_int64 sz = 0;
+    rc = pFile->pMethods->xFileSize(pFile, &sz);
+    if( rc==SQLITE_OK ){
+      p->iMark = sz - APND_MARK_SIZE;
+      if( iOfst + iAmt + p->iPgOne > p->iMark ){
+        p->iMark = p->iPgOne + iOfst + iAmt;
+        rc = apndWriteMark(p, pFile);
+      }
+    }
+  }
+  return rc;
+}
+
+/*
+** Truncate an apnd-file.
+*/
+static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
+  int rc;
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
+  if( rc==SQLITE_OK ){
+    p->iMark = p->iPgOne+size;
+    rc = apndWriteMark(p, pFile);
+  }
+  return rc;
+}
+
+/*
+** Sync an apnd-file.
+*/
+static int apndSync(sqlite3_file *pFile, int flags){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xSync(pFile, flags);
+}
+
+/*
+** Return the current file-size of an apnd-file.
+*/
+static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
+  ApndFile *p = (ApndFile *)pFile;
+  int rc;
+  pFile = ORIGFILE(p);
+  rc = pFile->pMethods->xFileSize(pFile, pSize);
+  if( rc==SQLITE_OK && p->iPgOne ){
+    *pSize -= p->iPgOne + APND_MARK_SIZE;
+  }
+  return rc;
+}
+
+/*
+** Lock an apnd-file.
+*/
+static int apndLock(sqlite3_file *pFile, int eLock){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xLock(pFile, eLock);
+}
+
+/*
+** Unlock an apnd-file.
+*/
+static int apndUnlock(sqlite3_file *pFile, int eLock){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xUnlock(pFile, eLock);
+}
+
+/*
+** Check if another file-handle holds a RESERVED lock on an apnd-file.
+*/
+static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xCheckReservedLock(pFile, pResOut);
+}
+
+/*
+** File control method. For custom operations on an apnd-file.
+*/
+static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){
+  ApndFile *p = (ApndFile *)pFile;
+  int rc;
+  pFile = ORIGFILE(pFile);
+  rc = pFile->pMethods->xFileControl(pFile, op, pArg);
+  if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
+    *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg);
+  }
+  return rc;
+}
+
+/*
+** Return the sector-size in bytes for an apnd-file.
+*/
+static int apndSectorSize(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xSectorSize(pFile);
+}
+
+/*
+** Return the device characteristic flags supported by an apnd-file.
+*/
+static int apndDeviceCharacteristics(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xDeviceCharacteristics(pFile);
+}
+
+/* Create a shared memory file mapping */
+static int apndShmMap(
+  sqlite3_file *pFile,
+  int iPg,
+  int pgsz,
+  int bExtend,
+  void volatile **pp
+){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp);
+}
+
+/* Perform locking on a shared-memory segment */
+static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmLock(pFile,offset,n,flags);
+}
+
+/* Memory barrier operation on shared memory */
+static void apndShmBarrier(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  pFile->pMethods->xShmBarrier(pFile);
+}
+
+/* Unmap a shared memory segment */
+static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmUnmap(pFile,deleteFlag);
+}
+
+/* Fetch a page of a memory-mapped file */
+static int apndFetch(
+  sqlite3_file *pFile,
+  sqlite3_int64 iOfst,
+  int iAmt,
+  void **pp
+){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
+}
+
+/* Release a memory-mapped page */
+static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
+}
+
+/*
+** Check to see if the file is an ordinary SQLite database file.
+*/
+static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
+  int rc;
+  char zHdr[16];
+  static const char aSqliteHdr[] = "SQLite format 3";
+  if( sz<512 ) return 0;
+  rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0);
+  if( rc ) return 0;
+  return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0;
+}
+
+/*
+** Try to read the append-mark off the end of a file.  Return the
+** start of the appended database if the append-mark is present.  If
+** there is no append-mark, return -1;
+*/
+static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){
+  int rc, i;
+  sqlite3_int64 iMark;
+  unsigned char a[APND_MARK_SIZE];
+
+  if( sz<=APND_MARK_SIZE ) return -1;
+  rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
+  if( rc ) return -1;
+  if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1;
+  iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56;
+  for(i=1; i<8; i++){    
+    iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i);
+  }
+  return iMark;
+}
+
+/*
+** Open an apnd file handle.
+*/
+static int apndOpen(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_file *pFile,
+  int flags,
+  int *pOutFlags
+){
+  ApndFile *p;
+  sqlite3_file *pSubFile;
+  sqlite3_vfs *pSubVfs;
+  int rc;
+  sqlite3_int64 sz;
+  pSubVfs = ORIGVFS(pVfs);
+  if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
+    return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
+  }
+  p = (ApndFile*)pFile;
+  memset(p, 0, sizeof(*p));
+  pSubFile = ORIGFILE(pFile);
+  p->base.pMethods = &apnd_io_methods;
+  rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
+  if( rc ) goto apnd_open_done;
+  rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
+  if( rc ){
+    pSubFile->pMethods->xClose(pSubFile);
+    goto apnd_open_done;
+  }
+  if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
+    memmove(pFile, pSubFile, pSubVfs->szOsFile);
+    return SQLITE_OK;
+  }
+  p->iMark = 0;
+  p->iPgOne = apndReadMark(sz, pFile);
+  if( p->iPgOne>0 ){
+    return SQLITE_OK;
+  }
+  if( (flags & SQLITE_OPEN_CREATE)==0 ){
+    pSubFile->pMethods->xClose(pSubFile);
+    rc = SQLITE_CANTOPEN;
+  }
+  p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
+apnd_open_done:
+  if( rc ) pFile->pMethods = 0;
+  return rc;
+}
+
+/*
+** All other VFS methods are pass-thrus.
+*/
+static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
+  return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync);
+}
+static int apndAccess(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int flags, 
+  int *pResOut
+){
+  return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut);
+}
+static int apndFullPathname(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int nOut, 
+  char *zOut
+){
+  return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut);
+}
+static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){
+  return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
+}
+static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
+  ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
+}
+static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
+  return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
+}
+static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){
+  ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
+}
+static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+  return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
+}
+static int apndSleep(sqlite3_vfs *pVfs, int nMicro){
+  return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
+}
+static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
+  return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
+}
+static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+  return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
+}
+static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
+  return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
+}
+static int apndSetSystemCall(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_syscall_ptr pCall
+){
+  return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall);
+}
+static sqlite3_syscall_ptr apndGetSystemCall(
+  sqlite3_vfs *pVfs,
+  const char *zName
+){
+  return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName);
+}
+static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
+  return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName);
+}
+
+  
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+/* 
+** This routine is called when the extension is loaded.
+** Register the new VFS.
+*/
+int sqlite3_appendvfs_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  sqlite3_vfs *pOrig;
+  SQLITE_EXTENSION_INIT2(pApi);
+  pOrig = sqlite3_vfs_find(0);
+  apnd_vfs.iVersion = pOrig->iVersion;
+  apnd_vfs.pAppData = pOrig;
+  apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile);
+  rc = sqlite3_vfs_register(&apnd_vfs, 0);
+#ifdef APPENDVFS_TEST
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);
+  }
+#endif
+  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
+  return rc;
+}

ext/misc/compress.c

@@ -27,6 +27,21 @@ SQLITE_EXTENSION_INIT1
 ** seven bits per integer stored in the lower seven bits of each byte.
 ** More significant bits occur first.  The most significant bit (0x80)
 ** is a flag to indicate the end of the integer.
+**
+** This function, SQLAR, and ZIP all use the same "deflate" compression
+** algorithm, but each is subtly different:
+**
+**   *  ZIP uses raw deflate.
+**
+**   *  SQLAR uses the "zlib format" which is raw deflate with a two-byte
+**      algorithm-identification header and a four-byte checksum at the end.
+**
+**   *  This utility uses the "zlib format" like SQLAR, but adds the variable-
+**      length integer uncompressed size value at the beginning.
+**
+** This function might be extended in the future to support compression
+** formats other than deflate, by providing a different algorithm-id
+** mark following the variable-length integer size parameter.
 */
 static void compressFunc(
   sqlite3_context *context,

ext/misc/fileio.c

@@ -261,15 +261,8 @@ static int writeFile(
   }
 
   if( mtime>=0 ){
-#if !defined(_WIN32) && !defined(WIN32)
-    struct timespec times[2];
-    times[0].tv_nsec = times[1].tv_nsec = 0;
-    times[0].tv_sec = time(0);
-    times[1].tv_sec = mtime;
-    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
-      return 1;
-    }
-#else
+#if defined(_WIN32)
+    /* Windows */
     FILETIME lastAccess;
     FILETIME lastWrite;
     SYSTEMTIME currentTime;
@@ -291,6 +284,24 @@ static int writeFile(
     }else{
       return 1;
     }
+#elif defined(AT_FDCWD)
+    /* Recent unix */
+    struct timespec times[2];
+    times[0].tv_nsec = times[1].tv_nsec = 0;
+    times[0].tv_sec = time(0);
+    times[1].tv_sec = mtime;
+    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
+      return 1;
+    }
+#else
+    /* Legacy unix */
+    struct timeval times[2];
+    times[0].tv_usec = times[1].tv_usec = 0;
+    times[0].tv_sec = time(0);
+    times[1].tv_sec = mtime;
+    if( utimes(zFile, times) ){
+      return 1;
+    }
 #endif
   }
 

ext/misc/normalize.c

@@ -0,0 +1,707 @@
+/*
+** 2018-01-08
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains code to implement the sqlite3_normalize() function.
+**
+**    char *sqlite3_normalize(const char *zSql);
+**
+** This function takes an SQL string as input and returns a "normalized"
+** version of that string in memory obtained from sqlite3_malloc64().  The
+** caller is responsible for ensuring that the returned memory is freed.
+**
+** If a memory allocation error occurs, this routine returns NULL.
+**
+** The normalization consists of the following transformations:
+**
+**   (1)  Convert every literal (string, blob literal, numeric constant,
+**        or "NULL" constant) into a ?
+**
+**   (2)  Remove all superfluous whitespace, including comments.  Change
+**        all required whitespace to a single space character.
+**
+**   (3)  Lowercase all ASCII characters.
+**
+**   (4)  If an IN or NOT IN operator is followed by a list of 1 or more
+**        values, convert that list into "(?,?,?)".
+**
+** The purpose of normalization is two-fold:
+**
+**   (1)  Sanitize queries by removing potentially private or sensitive
+**        information contained in literals.
+**
+**   (2)  Identify structurally identical queries by comparing their
+**        normalized forms.
+**
+** Command-Line Utility
+** --------------------
+**
+** This file also contains code for a command-line utility that converts
+** SQL queries in text files into their normalized forms.  To build the
+** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI
+** and link it against the SQLite library.
+*/
+#include <sqlite3.h>
+#include <string.h>
+
+/*
+** Implementation note:
+**
+** Much of the tokenizer logic is copied out of the tokenize.c source file
+** of SQLite.  That logic could be simplified for this particular application,
+** but that would impose a risk of introducing subtle errors.  It is best to
+** keep the code as close to the original as possible.
+**
+** The tokenize code is in sync with the SQLite core as of 2018-01-08.
+** Any future changes to the core tokenizer might require corresponding
+** adjustments to the tokenizer logic in this module.
+*/
+
+
+/* Character classes for tokenizing
+**
+** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
+** using a lookup table, whereas a switch() directly on c uses a binary search.
+** The lookup table is much faster.  To maximize speed, and to ensure that
+** a lookup table is used, all of the classes need to be small integers and
+** all of them need to be used within the switch.
+*/
+#define CC_X          0    /* The letter 'x', or start of BLOB literal */
+#define CC_KYWD       1    /* Alphabetics or '_'.  Usable in a keyword */
+#define CC_ID         2    /* unicode characters usable in IDs */
+#define CC_DIGIT      3    /* Digits */
+#define CC_DOLLAR     4    /* '$' */
+#define CC_VARALPHA   5    /* '@', '#', ':'.  Alphabetic SQL variables */
+#define CC_VARNUM     6    /* '?'.  Numeric SQL variables */
+#define CC_SPACE      7    /* Space characters */
+#define CC_QUOTE      8    /* '"', '\'', or '`'.  String literals, quoted ids */
+#define CC_QUOTE2     9    /* '['.   [...] style quoted ids */
+#define CC_PIPE      10    /* '|'.   Bitwise OR or concatenate */
+#define CC_MINUS     11    /* '-'.  Minus or SQL-style comment */
+#define CC_LT        12    /* '<'.  Part of < or <= or <> */
+#define CC_GT        13    /* '>'.  Part of > or >= */
+#define CC_EQ        14    /* '='.  Part of = or == */
+#define CC_BANG      15    /* '!'.  Part of != */
+#define CC_SLASH     16    /* '/'.  / or c-style comment */
+#define CC_LP        17    /* '(' */
+#define CC_RP        18    /* ')' */
+#define CC_SEMI      19    /* ';' */
+#define CC_PLUS      20    /* '+' */
+#define CC_STAR      21    /* '*' */
+#define CC_PERCENT   22    /* '%' */
+#define CC_COMMA     23    /* ',' */
+#define CC_AND       24    /* '&' */
+#define CC_TILDA     25    /* '~' */
+#define CC_DOT       26    /* '.' */
+#define CC_ILLEGAL   27    /* Illegal character */
+
+static const unsigned char aiClass[] = {
+/*         x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf */
+/* 0x */   27, 27, 27, 27, 27, 27, 27, 27, 27,  7,  7, 27,  7,  7, 27, 27,
+/* 1x */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* 2x */    7, 15,  8,  5,  4, 22, 24,  8, 17, 18, 21, 20, 23, 11, 26, 16,
+/* 3x */    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  5, 19, 12, 14, 13,  6,
+/* 4x */    5,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+/* 5x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1,  9, 27, 27, 27,  1,
+/* 6x */    8,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+/* 7x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1, 27, 10, 27, 25, 27,
+/* 8x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* 9x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Ax */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Bx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Cx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Dx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Ex */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Fx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2
+};
+
+/* An array to map all upper-case characters into their corresponding
+** lower-case character. 
+**
+** SQLite only considers US-ASCII (or EBCDIC) characters.  We do not
+** handle case conversions for the UTF character set since the tables
+** involved are nearly as big or bigger than SQLite itself.
+*/
+static const unsigned char sqlite3UpperToLower[] = {
+      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
+     18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+     36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
+     54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103,
+    104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,
+    122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107,
+    108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,
+    126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
+    144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,
+    162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,
+    180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,
+    198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,
+    216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,
+    234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,
+    252,253,254,255
+};
+
+/*
+** The following 256 byte lookup table is used to support SQLites built-in
+** equivalents to the following standard library functions:
+**
+**   isspace()                        0x01
+**   isalpha()                        0x02
+**   isdigit()                        0x04
+**   isalnum()                        0x06
+**   isxdigit()                       0x08
+**   toupper()                        0x20
+**   SQLite identifier character      0x40
+**   Quote character                  0x80
+**
+** Bit 0x20 is set if the mapped character requires translation to upper
+** case. i.e. if the character is a lower-case ASCII character.
+** If x is a lower-case ASCII character, then its upper-case equivalent
+** is (x - 0x20). Therefore toupper() can be implemented as:
+**
+**   (x & ~(map[x]&0x20))
+**
+** The equivalent of tolower() is implemented using the sqlite3UpperToLower[]
+** array. tolower() is used more often than toupper() by SQLite.
+**
+** Bit 0x40 is set if the character is non-alphanumeric and can be used in an 
+** SQLite identifier.  Identifiers are alphanumerics, "_", "$", and any
+** non-ASCII UTF character. Hence the test for whether or not a character is
+** part of an identifier is 0x46.
+*/
+static const unsigned char sqlite3CtypeMap[256] = {
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 00..07    ........ */
+  0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00,  /* 08..0f    ........ */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 10..17    ........ */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 18..1f    ........ */
+  0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80,  /* 20..27     !"#$%&' */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 28..2f    ()*+,-./ */
+  0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c,  /* 30..37    01234567 */
+  0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 38..3f    89:;<=>? */
+
+  0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02,  /* 40..47    @ABCDEFG */
+  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  /* 48..4f    HIJKLMNO */
+  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  /* 50..57    PQRSTUVW */
+  0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40,  /* 58..5f    XYZ[\]^_ */
+  0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22,  /* 60..67    `abcdefg */
+  0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,  /* 68..6f    hijklmno */
+  0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,  /* 70..77    pqrstuvw */
+  0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 78..7f    xyz{|}~. */
+
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 80..87    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 88..8f    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 90..97    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 98..9f    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* a0..a7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* a8..af    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* b0..b7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* b8..bf    ........ */
+
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* c0..c7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* c8..cf    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* d0..d7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* d8..df    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* e0..e7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* e8..ef    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* f0..f7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40   /* f8..ff    ........ */
+};
+#define sqlite3Toupper(x)   ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20))
+#define sqlite3Isspace(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x01)
+#define sqlite3Isalnum(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x06)
+#define sqlite3Isalpha(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x02)
+#define sqlite3Isdigit(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x04)
+#define sqlite3Isxdigit(x)  (sqlite3CtypeMap[(unsigned char)(x)]&0x08)
+#define sqlite3Tolower(x)   (sqlite3UpperToLower[(unsigned char)(x)])
+#define sqlite3Isquote(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x80)
+
+
+/*
+** If X is a character that can be used in an identifier then
+** IdChar(X) will be true.  Otherwise it is false.
+**
+** For ASCII, any character with the high-order bit set is
+** allowed in an identifier.  For 7-bit characters, 
+** sqlite3IsIdChar[X] must be 1.
+**
+** For EBCDIC, the rules are more complex but have the same
+** end result.
+**
+** Ticket #1066.  the SQL standard does not allow '$' in the
+** middle of identifiers.  But many SQL implementations do. 
+** SQLite will allow '$' in identifiers for compatibility.
+** But the feature is undocumented.
+*/
+#define IdChar(C)  ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
+
+/*
+** Ignore testcase() macros
+*/
+#define testcase(X)
+
+/*
+** Token values
+*/
+#define TK_SPACE    0
+#define TK_NAME     1
+#define TK_LITERAL  2
+#define TK_PUNCT    3
+#define TK_ERROR    4
+
+#define TK_MINUS    TK_PUNCT
+#define TK_LP       TK_PUNCT
+#define TK_RP       TK_PUNCT
+#define TK_SEMI     TK_PUNCT
+#define TK_PLUS     TK_PUNCT
+#define TK_STAR     TK_PUNCT
+#define TK_SLASH    TK_PUNCT
+#define TK_REM      TK_PUNCT
+#define TK_EQ       TK_PUNCT
+#define TK_LE       TK_PUNCT
+#define TK_NE       TK_PUNCT
+#define TK_LSHIFT   TK_PUNCT
+#define TK_LT       TK_PUNCT
+#define TK_GE       TK_PUNCT
+#define TK_RSHIFT   TK_PUNCT
+#define TK_GT       TK_PUNCT
+#define TK_GE       TK_PUNCT
+#define TK_BITOR    TK_PUNCT
+#define TK_CONCAT   TK_PUNCT
+#define TK_COMMA    TK_PUNCT
+#define TK_BITAND   TK_PUNCT
+#define TK_BITNOT   TK_PUNCT
+#define TK_STRING   TK_LITERAL
+#define TK_ID       TK_NAME
+#define TK_ILLEGAL  TK_ERROR
+#define TK_DOT      TK_PUNCT
+#define TK_INTEGER  TK_LITERAL
+#define TK_FLOAT    TK_LITERAL
+#define TK_VARIABLE TK_LITERAL
+#define TK_BLOB     TK_LITERAL
+
+/*
+** Return the length (in bytes) of the token that begins at z[0]. 
+** Store the token type in *tokenType before returning.
+*/
+static int sqlite3GetToken(const unsigned char *z, int *tokenType){
+  int i, c;
+  switch( aiClass[*z] ){  /* Switch on the character-class of the first byte
+                          ** of the token. See the comment on the CC_ defines
+                          ** above. */
+    case CC_SPACE: {
+      for(i=1; sqlite3Isspace(z[i]); i++){}
+      *tokenType = TK_SPACE;
+      return i;
+    }
+    case CC_MINUS: {
+      if( z[1]=='-' ){
+        for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
+        *tokenType = TK_SPACE;
+        return i;
+      }
+      *tokenType = TK_MINUS;
+      return 1;
+    }
+    case CC_LP: {
+      *tokenType = TK_LP;
+      return 1;
+    }
+    case CC_RP: {
+      *tokenType = TK_RP;
+      return 1;
+    }
+    case CC_SEMI: {
+      *tokenType = TK_SEMI;
+      return 1;
+    }
+    case CC_PLUS: {
+      *tokenType = TK_PLUS;
+      return 1;
+    }
+    case CC_STAR: {
+      *tokenType = TK_STAR;
+      return 1;
+    }
+    case CC_SLASH: {
+      if( z[1]!='*' || z[2]==0 ){
+        *tokenType = TK_SLASH;
+        return 1;
+      }
+      for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
+      if( c ) i++;
+      *tokenType = TK_SPACE;
+      return i;
+    }
+    case CC_PERCENT: {
+      *tokenType = TK_REM;
+      return 1;
+    }
+    case CC_EQ: {
+      *tokenType = TK_EQ;
+      return 1 + (z[1]=='=');
+    }
+    case CC_LT: {
+      if( (c=z[1])=='=' ){
+        *tokenType = TK_LE;
+        return 2;
+      }else if( c=='>' ){
+        *tokenType = TK_NE;
+        return 2;
+      }else if( c=='<' ){
+        *tokenType = TK_LSHIFT;
+        return 2;
+      }else{
+        *tokenType = TK_LT;
+        return 1;
+      }
+    }
+    case CC_GT: {
+      if( (c=z[1])=='=' ){
+        *tokenType = TK_GE;
+        return 2;
+      }else if( c=='>' ){
+        *tokenType = TK_RSHIFT;
+        return 2;
+      }else{
+        *tokenType = TK_GT;
+        return 1;
+      }
+    }
+    case CC_BANG: {
+      if( z[1]!='=' ){
+        *tokenType = TK_ILLEGAL;
+        return 1;
+      }else{
+        *tokenType = TK_NE;
+        return 2;
+      }
+    }
+    case CC_PIPE: {
+      if( z[1]!='|' ){
+        *tokenType = TK_BITOR;
+        return 1;
+      }else{
+        *tokenType = TK_CONCAT;
+        return 2;
+      }
+    }
+    case CC_COMMA: {
+      *tokenType = TK_COMMA;
+      return 1;
+    }
+    case CC_AND: {
+      *tokenType = TK_BITAND;
+      return 1;
+    }
+    case CC_TILDA: {
+      *tokenType = TK_BITNOT;
+      return 1;
+    }
+    case CC_QUOTE: {
+      int delim = z[0];
+      testcase( delim=='`' );
+      testcase( delim=='\'' );
+      testcase( delim=='"' );
+      for(i=1; (c=z[i])!=0; i++){
+        if( c==delim ){
+          if( z[i+1]==delim ){
+            i++;
+          }else{
+            break;
+          }
+        }
+      }
+      if( c=='\'' ){
+        *tokenType = TK_STRING;
+        return i+1;
+      }else if( c!=0 ){
+        *tokenType = TK_ID;
+        return i+1;
+      }else{
+        *tokenType = TK_ILLEGAL;
+        return i;
+      }
+    }
+    case CC_DOT: {
+      if( !sqlite3Isdigit(z[1]) ){
+        *tokenType = TK_DOT;
+        return 1;
+      }
+      /* If the next character is a digit, this is a floating point
+      ** number that begins with ".".  Fall thru into the next case */
+    }
+    case CC_DIGIT: {
+      *tokenType = TK_INTEGER;
+      if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
+        for(i=3; sqlite3Isxdigit(z[i]); i++){}
+        return i;
+      }
+      for(i=0; sqlite3Isdigit(z[i]); i++){}
+      if( z[i]=='.' ){
+        i++;
+        while( sqlite3Isdigit(z[i]) ){ i++; }
+        *tokenType = TK_FLOAT;
+      }
+      if( (z[i]=='e' || z[i]=='E') &&
+           ( sqlite3Isdigit(z[i+1]) 
+            || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
+           )
+      ){
+        i += 2;
+        while( sqlite3Isdigit(z[i]) ){ i++; }
+        *tokenType = TK_FLOAT;
+      }
+      while( IdChar(z[i]) ){
+        *tokenType = TK_ILLEGAL;
+        i++;
+      }
+      return i;
+    }
+    case CC_QUOTE2: {
+      for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
+      *tokenType = c==']' ? TK_ID : TK_ILLEGAL;
+      return i;
+    }
+    case CC_VARNUM: {
+      *tokenType = TK_VARIABLE;
+      for(i=1; sqlite3Isdigit(z[i]); i++){}
+      return i;
+    }
+    case CC_DOLLAR:
+    case CC_VARALPHA: {
+      int n = 0;
+      testcase( z[0]=='$' );  testcase( z[0]=='@' );
+      testcase( z[0]==':' );  testcase( z[0]=='#' );
+      *tokenType = TK_VARIABLE;
+      for(i=1; (c=z[i])!=0; i++){
+        if( IdChar(c) ){
+          n++;
+        }else if( c=='(' && n>0 ){
+          do{
+            i++;
+          }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' );
+          if( c==')' ){
+            i++;
+          }else{
+            *tokenType = TK_ILLEGAL;
+          }
+          break;
+        }else if( c==':' && z[i+1]==':' ){
+          i++;
+        }else{
+          break;
+        }
+      }
+      if( n==0 ) *tokenType = TK_ILLEGAL;
+      return i;
+    }
+    case CC_KYWD: {
+      for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
+      if( IdChar(z[i]) ){
+        /* This token started out using characters that can appear in keywords,
+        ** but z[i] is a character not allowed within keywords, so this must
+        ** be an identifier instead */
+        i++;
+        break;
+      }
+      *tokenType = TK_ID;
+      return i;
+    }
+    case CC_X: {
+      testcase( z[0]=='x' ); testcase( z[0]=='X' );
+      if( z[1]=='\'' ){
+        *tokenType = TK_BLOB;
+        for(i=2; sqlite3Isxdigit(z[i]); i++){}
+        if( z[i]!='\'' || i%2 ){
+          *tokenType = TK_ILLEGAL;
+          while( z[i] && z[i]!='\'' ){ i++; }
+        }
+        if( z[i] ) i++;
+        return i;
+      }
+      /* If it is not a BLOB literal, then it must be an ID, since no
+      ** SQL keywords start with the letter 'x'.  Fall through */
+    }
+    case CC_ID: {
+      i = 1;
+      break;
+    }
+    default: {
+      *tokenType = TK_ILLEGAL;
+      return 1;
+    }
+  }
+  while( IdChar(z[i]) ){ i++; }
+  *tokenType = TK_ID;
+  return i;
+}
+
+char *sqlite3_normalize(const char *zSql){
+  char *z;              /* The output string */
+  sqlite3_int64 nZ;     /* Size of the output string in bytes */
+  sqlite3_int64 nSql;   /* Size of the input string in bytes */
+  int i;                /* Next character to read from zSql[] */
+  int j;                /* Next slot to fill in on z[] */
+  int tokenType;        /* Type of the next token */
+  int n;                /* Size of the next token */
+  int k;                /* Loop counter */
+
+  nSql = strlen(zSql);
+  nZ = nSql;
+  z = sqlite3_malloc64( nZ+2 );
+  if( z==0 ) return 0;
+  for(i=j=0; zSql[i]; i += n){
+    n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
+    switch( tokenType ){
+      case TK_SPACE: {
+        break;
+      }
+      case TK_ERROR: {
+        sqlite3_free(z);
+        return 0;
+      }
+      case TK_LITERAL: {
+        z[j++] = '?';
+        break;
+      }
+      case TK_PUNCT:
+      case TK_NAME: {
+        if( n==4 && sqlite3_strnicmp(zSql+i,"NULL",4)==0 ){
+          if( (j>=3 && strncmp(z+j-2,"is",2)==0 && !IdChar(z[j-3]))
+           || (j>=4 && strncmp(z+j-3,"not",3)==0 && !IdChar(z[j-4]))
+          ){
+            /* NULL is a keyword in this case, not a literal value */
+          }else{
+            /* Here the NULL is a literal value */
+            z[j++] = '?';
+            break;
+          }
+        }
+        if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' ';
+        for(k=0; k<n; k++){
+          z[j++] = sqlite3Tolower(zSql[i+k]);
+        }
+        break;
+      }
+    }
+  }
+  while( j>0 && z[j-1]==' ' ){ j--; }
+  if( i>0 && z[j-1]!=';' ){ z[j++] = ';'; }
+  z[j] = 0;
+
+  /* Make a second pass converting "in(...)" where the "..." is not a
+  ** SELECT statement into "in(?,?,?)" */
+  for(i=0; i<j; i=n){
+    char *zIn = strstr(z+i, "in(");
+    int nParen;
+    if( zIn==0 ) break;
+    n = (int)(zIn-z)+3;  /* Index of first char past "in(" */
+    if( n && IdChar(zIn[-1]) ) continue;
+    if( strncmp(zIn, "in(select",9)==0 && !IdChar(zIn[9]) ) continue;
+    if( strncmp(zIn, "in(with",7)==0 && !IdChar(zIn[7]) ) continue;
+    for(nParen=1, k=0; z[n+k]; k++){
+      if( z[n+k]=='(' ) nParen++;
+      if( z[n+k]==')' ){
+        nParen--;
+        if( nParen==0 ) break;
+      }
+    }
+    /* k is the number of bytes in the "..." within "in(...)" */
+    if( k<5 ){
+      z = sqlite3_realloc64(z, j+(5-k)+1);
+      if( z==0 ) return 0;
+      memmove(z+n+5, z+n+k, j-(n+k));
+    }else if( k>5 ){
+      memmove(z+n+5, z+n+k, j-(n+k));
+    }
+    j = j-k+5;
+    z[j] = 0;
+    memcpy(z+n, "?,?,?", 5);
+  }
+  return z;
+}
+
+/*
+** For testing purposes, or to build a stand-alone SQL normalizer program,
+** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link
+** it against any SQLite library.  The resulting command-line program will
+** run sqlite3_normalize() over the text of all files named on the command-
+** line and show the result on standard output.
+*/
+#ifdef SQLITE_NORMALIZE_CLI
+#include <stdio.h>
+#include <stdlib.h>
+
+/*
+** Break zIn up into separate SQL statements and run sqlite3_normalize()
+** on each one.  Print the result of each run.
+*/
+static void normalizeFile(char *zIn){
+  int i;
+  if( zIn==0 ) return;
+  for(i=0; zIn[i]; i++){
+    char cSaved;
+    if( zIn[i]!=';' ) continue;
+    cSaved = zIn[i+1];
+    zIn[i+1] = 0;
+    if( sqlite3_complete(zIn) ){
+      char *zOut = sqlite3_normalize(zIn);
+      if( zOut ){
+        printf("%s\n", zOut);
+        sqlite3_free(zOut);
+      }else{
+        fprintf(stderr, "ERROR: %s\n", zIn);
+      }
+      zIn[i+1] = cSaved;
+      zIn += i+1;
+      i = -1;
+    }else{
+      zIn[i+1] = cSaved;
+    }
+  }
+}
+
+/*
+** The main routine for "sql_normalize".  Read files named on the
+** command-line and run the text of each through sqlite3_normalize().
+*/
+int main(int argc, char **argv){
+  int i;
+  FILE *in;
+  char *zBuf = 0;
+  sqlite3_int64 sz, got;
+
+  for(i=1; i<argc; i++){
+    in = fopen(argv[i], "rb");
+    if( in==0 ){
+      fprintf(stderr, "cannot open \"%s\"\n", argv[i]);
+      continue;
+    }
+    fseek(in, 0, SEEK_END);
+    sz = ftell(in);
+    rewind(in);
+    zBuf = sqlite3_realloc64(zBuf, sz+1);
+    if( zBuf==0 ){
+      fprintf(stderr, "failed to malloc for %lld bytes\n", sz);
+      exit(1);
+    }
+    got = fread(zBuf, 1, sz, in);
+    fclose(in);
+    if( got!=sz ){
+      fprintf(stderr, "only able to read %lld of %lld bytes from \"%s\"\n",
+              got, sz, argv[i]);
+    }else{
+      zBuf[got] = 0;
+      normalizeFile(zBuf);
+    }
+  }
+  sqlite3_free(zBuf);
+}
+#endif /* SQLITE_NORMALIZE_CLI */

ext/misc/sqlar.c

@@ -24,6 +24,14 @@ SQLITE_EXTENSION_INIT1
 ** If the type of X is SQLITE_BLOB, and compressing that blob using
 ** zlib utility function compress() yields a smaller blob, return the
 ** compressed blob. Otherwise, return a copy of X.
+**
+** SQLar uses the "zlib format" for compressed content.  The zlib format
+** contains a two-byte identification header and a four-byte checksum at
+** the end.  This is different from ZIP which uses the raw deflate format.
+**
+** Future enhancements to SQLar might add support for new compression formats.
+** If so, those new formats will be identified by alternative headers in the
+** compressed data.
 */
 static void sqlarCompressFunc(
   sqlite3_context *context,

main.mk

@@ -361,6 +361,7 @@ TESTSRC += \
   $(TOP)/ext/misc/ieee754.c \
   $(TOP)/ext/misc/mmapwarm.c \
   $(TOP)/ext/misc/nextchar.c \
+  $(TOP)/ext/misc/normalize.c \
   $(TOP)/ext/misc/percentile.c \
   $(TOP)/ext/misc/regexp.c \
   $(TOP)/ext/misc/remember.c \
@@ -487,7 +488,8 @@ TESTPROGS = \
   sqlite3_analyzer$(EXE) \
   sqlite3_checker$(EXE) \
   sqldiff$(EXE) \
-  dbhash$(EXE)
+  dbhash$(EXE) \
+  sqltclsh$(EXE)
 
 # Databases containing fuzzer test cases
 #
@@ -693,6 +695,7 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 # Source files that go into making shell.c
 SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
+        $(TOP)/ext/misc/appendvfs.c \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
 	$(TOP)/ext/misc/completion.c \
@@ -819,6 +822,12 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $
 sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl
+	tclsh $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
+
+sqltclsh$(EXE): sqltclsh.c
+	$(TCCX) $(TCL_FLAGS) sqltclsh.c -o $@ $(LIBTCL) $(THREADLIB) 
+
 sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
 	$(TCCX) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert$(EXE) $(THREADLIB)
 

manifest

@@ -1,10 +1,10 @@
-C Fix\sa\sbug\sintroduced\sby\sefforts\sto\sreduce\scompiler\swarnings\sin\ncheck-in\s[364ac333b030f0]
-D 2018-01-05T18:51:25.650
+C Test\scases\sfor\ssqlite3_normalize().
+D 2018-01-08T20:04:57.992
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in f5c6285ac43b2e567d6cf463dff3744da960a19f2cf141744b4472842a97681e
+F Makefile.in 822052988bbf6204ca2c3e2665618b5a48acec24105d5afa8c4ebb59cb941d3e
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 2335db317697a3ea33ec3a258be9f46c973fcfd4e7cffe6798c6f3d55bc83d5b
+F Makefile.msc 0f2875c7d4cb93ae4dab661f58f9d68699d323cfb87ebd7d8dccbb9949979b83
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -13,7 +13,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6220fb39
-F autoconf/Makefile.msc 2b4b5e5ff7e7a9806ebdd4b441f8fb54695a3628701a97ae53860e67e872acc3
+F autoconf/Makefile.msc 2c50a59319af7da4eaca8c13e3240881b1bc245fd175845a055faab7d03d6e67
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
 F autoconf/configure.ac 8dd08ca564279fff091c9bfdd2599d8f992c9f1f70c5396de2126ad2bd1b3bed
@@ -223,7 +223,7 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
 F ext/lsm1/Makefile 98b0a24b45e248283d6bea4b6cb3e58d7b394edd8e96a0ac28c5fa5104813bad
 F ext/lsm1/Makefile.msc f8c878b467232226de288da320e1ac71c131f5ec91e08b21f502303347260013
 F ext/lsm1/lsm-test/README 87ea529d2abe615e856d4714bfe8bb185e6c2771b8612aa6298588b7b43e6f86
-F ext/lsm1/lsm-test/lsmtest.h 5847594d4b43ec3412e1fd97104f7eb5fd770be55e691e6cb2e80929f86bebe3
+F ext/lsm1/lsm-test/lsmtest.h cf58528ffe0cfe535e91b44584e2ec5fb1caacdabecef0d8dcf83bf83168bf28
 F ext/lsm1/lsm-test/lsmtest1.c 33158978327f800e82b6a47c09b86ace809f56a9ff10b0162273ec1186cc3153
 F ext/lsm1/lsm-test/lsmtest2.c 188b09aec776516aeedcfd13b9c6faf85ba16b3671a0897a2c740ee00a5dc4f8
 F ext/lsm1/lsm-test/lsmtest3.c 9ab87528a36dbf4a61d7c8ad954f5ee368c0878c127b84b942b2e2abe522de26
@@ -239,10 +239,10 @@ F ext/lsm1/lsm-test/lsmtest_func.c 159aa401bc8032bfa3d8cf2977bd687abebab88025589
 F ext/lsm1/lsm-test/lsmtest_io.c cf11b27b129c6bd5818fa1d440176502dc27229f0db892b4479118d61993ea20
 F ext/lsm1/lsm-test/lsmtest_main.c a9bc647738c0dcaebf205d6d194b3ce4a6ef3925801cd2d919f0a4ea33a15aeb
 F ext/lsm1/lsm-test/lsmtest_mem.c 4e63c764345ab1df59d4f13a77980c6f3643798210b10d6cdbd785b4b888fda5
-F ext/lsm1/lsm-test/lsmtest_tdb.c 555fb101d2fe638abdd133e9294536c857fb38e0f227e049c00a67f51eaece06
+F ext/lsm1/lsm-test/lsmtest_tdb.c 618a8619183fda4f5540fcde15f9068293c5e3180e1a246e34409b0c148758b3
 F ext/lsm1/lsm-test/lsmtest_tdb.h 8733eee249b12956a9df8322994b43d19bd8c02ad2e8b0bb5164db4d6ccc1735
 F ext/lsm1/lsm-test/lsmtest_tdb2.cc 99ea7f2dd9c7536c8fb9bdd329e4cfeb76899f3ddf6f48bdd3926e016922b715
-F ext/lsm1/lsm-test/lsmtest_tdb3.c e44bf94e8bd724cd6ac161fd2cd44ffe43193932ad3a6bee1b07d80bb74012bb
+F ext/lsm1/lsm-test/lsmtest_tdb3.c b4e46b1d2fec553fe4efb44e341b43abd20556fde610db0cfffdc2300b72defe
 F ext/lsm1/lsm-test/lsmtest_tdb4.c 47e8bb5eba266472d690fb8264f1855ebdba0ae5a0e541e35fcda61ebf1d277f
 F ext/lsm1/lsm-test/lsmtest_util.c 241622db5a332a09c8e6e7606b617d288a37b557f7d3bce0bb97809f67cc2806
 F ext/lsm1/lsm-test/lsmtest_win32.c 0e0a224674c4d3170631c41b026b56c7e1672b151f5261e1b4cc19068641da2d
@@ -251,11 +251,11 @@ F ext/lsm1/lsmInt.h 5983690e05e83653cc01ba9d8fbf8455e534ddf8349ed9adedbf46a75497
 F ext/lsm1/lsm_ckpt.c 0eabfaf812ddb4ea43add38f05e430694cd054eb622c3e35af4c43118a2d5321
 F ext/lsm1/lsm_file.c 3c51841d5b3e7da162693cbac9a9f47eeedf6bcbbe2969a4d25e30c428c9fe36
 F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709d9a7c
-F ext/lsm1/lsm_main.c 801295038b548ae2e5fae93f08c3f945154f40848a03ff26b16eab5d04ba573a
+F ext/lsm1/lsm_main.c b5703f8042e71d3a2d65e671f6832e077e79e89e9975818f67f969922618db63
 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a
 F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea
 F ext/lsm1/lsm_shared.c 76adfc1ed9ffebaf92746dde4b370ccc48143ca8b05b563816eadd2aadf1c525
-F ext/lsm1/lsm_sorted.c d07ff7c28758542b8b4da4b5a1fb67b22a4d33e50e7f684cffe1f6c45cf5182c
+F ext/lsm1/lsm_sorted.c df7b393d9e4b85e6ad07181c6434d58a69a9145ee925d89c3ee1d2a210adaf9a
 F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82
 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb
 F ext/lsm1/lsm_unix.c 57361bcf5b1a1a028f5d66571ee490e9064d2cfb145a2cc9e5ddade467bb551b
@@ -268,21 +268,23 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
+F ext/misc/appendvfs.c 4c65f0b79686ae5a483134233d7fd912f0f2d4fd76023404f96f2290fff13b19
 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
-F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
+F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c f16e2ef87928b768883b259b6c53a67076e98e8daa0bbb624910406d59630b26
+F ext/misc/fileio.c 672e89e125846848e07e025cbe601dc370a93433d571d9422541a9cefbd12b76
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
 F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567
 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
+F ext/misc/normalize.c 19262ef3ef29d4de2f281b423326865c8916c63d0cb09f1dc98d24d5c1e8ba64
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
@@ -293,7 +295,7 @@ F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac
-F ext/misc/sqlar.c d355cd8b6e7280d2f61d4737672922acb512a2ab1cee52399ffb88980476e31c
+F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/unionvtab.c 2aa94902ea646e1aaf6c05eac944a14276cddd67735b2ad856030ffffbb6626c
@@ -405,7 +407,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 0c0ed98340ad4b7ebdfe587144a13cb12dfe3f09c16a6af5b4d4a7f3a881f2f8
+F main.mk c6b8ee2fa871e01ad6a460d750cd074010292464a60a37a2207930ae0df629dc
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -483,7 +485,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 17fc28661aae277767db63fa90e644b8c9bf1242fbd167dcc2e7af0f0e620bb7
+F src/shell.c.in 0e20ac49a6c4458620084bf7715ad39cd9557d9116e447548d2282000e8ca361
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -492,7 +494,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696
-F src/test1.c b52f9e7fe62016d357c3266fcfa0793cc1883d3cb2b11dfa39fcba2e70b0305c
+F src/test1.c 1ab7cbbb6693e08364c1a9241e2aee17f8c4925e4cc52396be77ae6845a05828
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -540,8 +542,8 @@ F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb26
 F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858
 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
-F src/test_windirent.c 17f91f5f2aa1bb7328abb49414c363b5d2a9d3ff
-F src/test_windirent.h 5afeb2b1e2920d5149573a4c30a69cba91f2e8a80e00941b738036fca39aca61
+F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1
+F src/test_windirent.h 8782864172ba5ae52c5c313c70faeadb324ff74de9c3dcc6b56a557dccaa1de6
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
@@ -1091,6 +1093,7 @@ F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660
 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1
 F test/nockpt.test 9a436a7213ba5ef7a32304998d386d3ea3f76c9d
 F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e
+F test/normalize.test 501630ab49b0b26b65c74124bf03e3374c1b57fa97aae750f84803609141d167
 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf
 F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161
 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934
@@ -1151,7 +1154,7 @@ F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8
 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8
 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c
 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254
-F test/releasetest.tcl 0b0b3d926e36822ff63b405d683544ce1014303b029f2678bbcf40c162b5f246 x
+F test/releasetest.tcl 6aaa853f7a7bbdc458d4cb42c0425228729b0f3e5769e9b41088c08eee999a49 x
 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb
 F test/rollback.test f580934279800d480a19176c6b44909df31ce7ad45267ea475a541daa522f3d3
 F test/rollback2.test 8435d6ff0f13f51d2a4181c232e706005fa90fc5
@@ -1597,7 +1600,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test d4f342ca918fd4d7981a12c1523f5041074cc592f25fecce4ee11446cc984f56
+F test/zipfile.test a1dd0429294cb9487900fc2b29aa9921329f20a7314aa0921b668246172ac090
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1630,7 +1633,7 @@ F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c
 F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd
-F tool/mkshellc.tcl 574307265b49d813301fba91ccd74e6a26d33f65f74b6891c320a0ffbee07895
+F tool/mkshellc.tcl 1f45770aea226ac093a9c72f718efbb88a2a2833409ec2e1c4cecae4202626f5
 F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
@@ -1663,6 +1666,8 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c
 F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb
 F tool/sqlite3_analyzer.c.in 7eeaae8b0d7577662acaabbb11107af0659d1b41bc1dfdd4d91422de27127968
+F tool/sqltclsh.c.in 8b2529b6c3cdd8ad6aaff21e80eb58370c428cb207b4607d0da4def064dcec56
+F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1694,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5c6d0a1d58533feb4ea6926f4d3611664a275ad216492b424056276da38e84a2
-R ed8496ab8972efeecd73ab10aa079f0a
+P db5d138e97f22ad4d4d11dbef96df93696ba0e557809066bc263ca3c3898f349
+R 5585a6831c91176202d5e0f81c59c944
 U drh
-Z ab3dd049af31828f3cc751c8ec71ac20
+Z ca698384a23dd84f9bb875fa9da10ba1

manifest.uuid

@@ -1 +1 @@
-1d6cee9ad448b10e69f351ef9dbec09110c5b189cba8734e637f41abe8f35bf8
+658f42257d56a3562dfa8e55023c6e497f55c565cd751d942de3a9e0cb2fa708

src/shell.c.in

@@ -61,6 +61,9 @@
 #include <stdio.h>
 #include <assert.h>
 #include "sqlite3.h"
+typedef sqlite3_int64 i64;
+typedef sqlite3_uint64 u64;
+typedef unsigned char u8;
 #if SQLITE_USER_AUTHENTICATION
 # include "sqlite3userauth.h"
 #endif
@@ -72,7 +75,16 @@
 # if !defined(__RTP__) && !defined(_WRS_KERNEL)
 #  include <pwd.h>
 # endif
+#endif
+#if (!defined(_WIN32) && !defined(WIN32)) || defined(__MINGW_H)
 # include <unistd.h>
+# include <dirent.h>
+# if defined(__MINGW_H)
+#  define DIRENT dirent
+#  ifndef S_ISLNK
+#   define S_ISLNK(mode) (0)
+#  endif
+# endif
 #endif
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -875,14 +887,15 @@ static void shellAddSchemaName(
 #define SQLITE_EXTENSION_INIT1
 #define SQLITE_EXTENSION_INIT2(X) (void)(X)
 
-#if defined(_WIN32) || defined(WIN32)
+#if defined(_WIN32) && defined(_MSC_VER)
+INCLUDE test_windirent.h
 INCLUDE test_windirent.c
 #define dirent DIRENT
-#define timespec TIMESPEC
 #endif
 INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
+INCLUDE ../ext/misc/appendvfs.c
 #ifdef SQLITE_HAVE_ZLIB
 INCLUDE ../ext/misc/zipfile.c
 INCLUDE ../ext/misc/sqlar.c
@@ -928,10 +941,11 @@ struct ExpertInfo {
 typedef struct ShellState ShellState;
 struct ShellState {
   sqlite3 *db;           /* The database */
-  int autoExplain;       /* Automatically turn on .explain mode */
-  int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
-  int statsOn;           /* True to display memory stats before each finalize */
-  int scanstatsOn;       /* True to display scan stats before each finalize */
+  u8 autoExplain;        /* Automatically turn on .explain mode */
+  u8 autoEQP;            /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
+  u8 statsOn;            /* True to display memory stats before each finalize */
+  u8 scanstatsOn;        /* True to display scan stats before each finalize */
+  u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
   int outCount;          /* Revert to stdout when reaching zero */
   int cnt;               /* Number of records displayed so far */
   FILE *out;             /* Write results here */
@@ -968,6 +982,7 @@ struct ShellState {
   ExpertInfo expert;        /* Valid if previous command was ".expert OPT..." */
 };
 
+
 /* Allowed values for ShellState.autoEQP
 */
 #define AUTOEQP_off      0
@@ -975,6 +990,13 @@ struct ShellState {
 #define AUTOEQP_trigger  2
 #define AUTOEQP_full     3
 
+/* Allowed values for ShellState.openMode
+*/
+#define SHELL_OPEN_UNSPEC     0      /* No open-mode specified */
+#define SHELL_OPEN_NORMAL     1      /* Normal database file */
+#define SHELL_OPEN_APPENDVFS  2      /* Use appendvfs */
+#define SHELL_OPEN_ZIPFILE    3      /* Use the zipfile virtual table */
+
 /*
 ** These are the allowed shellFlgs values
 */
@@ -1401,6 +1423,22 @@ static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){
   z[n] = c;
 }
 
+/*
+** Return true if string z[] has nothing but whitespace and comments to the
+** end of the first line.
+*/
+static int wsToEol(const char *z){
+  int i;
+  for(i=0; z[i]; i++){
+    if( z[i]=='\n' ) return 1;
+    if( IsSpace(z[i]) ) continue;
+    if( z[i]=='-' && z[i+1]=='-' ) return 1;
+    return 0;
+  }
+  return 1;
+}
+    
+
 /*
 ** This is the callback routine that the shell
 ** invokes for each row of a query result.
@@ -1540,13 +1578,15 @@ static int shell_callback(
       while( j>0 && IsSpace(z[j-1]) ){ j--; }
       z[j] = 0;
       if( strlen30(z)>=79 ){
-        for(i=j=0; (c = z[i])!=0; i++){
+        for(i=j=0; (c = z[i])!=0; i++){  /* Copy changes from z[i] back to z[j] */
           if( c==cEnd ){
             cEnd = 0;
           }else if( c=='"' || c=='\'' || c=='`' ){
             cEnd = c;
           }else if( c=='[' ){
             cEnd = ']';
+          }else if( c=='-' && z[i+1]=='-' ){
+            cEnd = '\n';
           }else if( c=='(' ){
             nParen++;
           }else if( c==')' ){
@@ -1557,7 +1597,9 @@ static int shell_callback(
             }
           }
           z[j++] = c;
-          if( nParen==1 && (c=='(' || c==',' || c=='\n') ){
+          if( nParen==1 && cEnd==0
+           && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1)))
+          ){
             if( c=='\n' ) j--;
             printSchemaLineN(p->out, z, j, "\n  ");
             j = 0;
@@ -2865,6 +2907,9 @@ static int run_schema_dump_query(
 ** Text of a help message
 */
 static char zHelp[] =
+#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE)
+  ".archive ...           Manage SQL archives: \".archive --help\" for details\n"
+#endif
 #ifndef SQLITE_OMIT_AUTHORIZATION
   ".auth ON|OFF           Show authorizer callbacks\n"
 #endif
@@ -3068,6 +3113,32 @@ static int session_filter(void *pCtx, const char *zTab){
 }
 #endif
 
+/*
+** Try to deduce the type of file for zName based on its content.  Return
+** one of the SHELL_OPEN_* constants.
+*/
+static int deduceDatabaseType(const char *zName){
+  FILE *f = fopen(zName, "rb");
+  size_t n;
+  int rc = SHELL_OPEN_UNSPEC;
+  char zBuf[100];
+  if( f==0 ) return SHELL_OPEN_NORMAL;
+  fseek(f, -25, SEEK_END);
+  n = fread(zBuf, 25, 1, f);
+  if( n==1 && memcmp(zBuf, "Start-Of-SQLite3-", 17)==0 ){
+    rc = SHELL_OPEN_APPENDVFS;
+  }else{
+    fseek(f, -22, SEEK_END);
+    n = fread(zBuf, 22, 1, f);
+    if( n==1 && zBuf[0]==0x50 && zBuf[1]==0x4b && zBuf[2]==0x05
+       && zBuf[3]==0x06 ){
+      rc = SHELL_OPEN_ZIPFILE;
+    }
+  }
+  fclose(f);
+  return rc;  
+}
+
 /*
 ** Make sure the database is open.  If it is not, then open it.  If
 ** the database fails to open, print an error message and exit.
@@ -3075,7 +3146,25 @@ static int session_filter(void *pCtx, const char *zTab){
 static void open_db(ShellState *p, int keepAlive){
   if( p->db==0 ){
     sqlite3_initialize();
-    sqlite3_open(p->zDbFilename, &p->db);
+    if( p->openMode==SHELL_OPEN_UNSPEC && access(p->zDbFilename,0)==0 ){
+      p->openMode = deduceDatabaseType(p->zDbFilename);
+    }
+    switch( p->openMode ){
+      case SHELL_OPEN_APPENDVFS: {
+        sqlite3_open_v2(p->zDbFilename, &p->db, 
+           SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
+        break;
+      }
+      case SHELL_OPEN_ZIPFILE: {
+        sqlite3_open(":memory:", &p->db);
+        break;
+      }
+      case SHELL_OPEN_UNSPEC:
+      case SHELL_OPEN_NORMAL: {
+        sqlite3_open(p->zDbFilename, &p->db);
+        break;
+      }
+    }
     globalDb = p->db;
     if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
       utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n",
@@ -3097,6 +3186,12 @@ static void open_db(ShellState *p, int keepAlive){
                             shellAddSchemaName, 0, 0);
     sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0,
                             shellModuleSchema, 0, 0);
+    if( p->openMode==SHELL_OPEN_ZIPFILE ){
+      char *zSql = sqlite3_mprintf(
+         "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
+      sqlite3_exec(p->db, zSql, 0, 0, 0);
+      sqlite3_free(zSql);
+    }
   }
 }
 
@@ -4266,6 +4361,10 @@ static int lintDotCommand(
   return SQLITE_ERROR;
 }
 
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
+/*********************************************************************************
+** The ".archive" or ".ar" command.
+*/
 static void shellPrepare(
   sqlite3 *db, 
   int *pRc, 
@@ -4335,8 +4434,7 @@ static void shellReset(
     *pRc = rc;
   }
 }
-
-/* 
+/*
 ** Structure representing a single ".ar" command.
 */
 typedef struct ArCommand ArCommand;
@@ -4828,13 +4926,13 @@ static int arCreateUpdate(
 ){
   const char *zSql = "SELECT name, mode, mtime, data FROM fsdir(?, ?)";
   const char *zCreate = 
-      "CREATE TABLE IF NOT EXISTS sqlar("
-      "name TEXT PRIMARY KEY,  -- name of the file\n"
-      "mode INT,               -- access permissions\n"
-      "mtime INT,              -- last modification time\n"
-      "sz INT,                 -- original file size\n"
-      "data BLOB               -- compressed content\n"
-  ")";
+      "CREATE TABLE IF NOT EXISTS sqlar(\n"
+      "  name TEXT PRIMARY KEY,  -- name of the file\n"
+      "  mode INT,               -- access permissions\n"
+      "  mtime INT,              -- last modification time\n"
+      "  sz INT,                 -- original file size\n"
+      "  data BLOB               -- compressed content\n"
+      ")";
   const char *zDrop = "DROP TABLE IF EXISTS sqlar";
   const char *zInsert = "REPLACE INTO sqlar VALUES(?,?,?,?,sqlar_compress(?))";
 
@@ -5004,6 +5102,9 @@ static int arDotCommand(
 
   return rc;
 }
+/* End of the ".archive" or ".ar" command logic
+**********************************************************************************/
+#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */
 
 /*
 ** Implementation of ".expert" dot command.
@@ -5126,8 +5227,8 @@ static int do_meta_command(char *zLine, ShellState *p){
   }else
 #endif
 
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-  if( c=='a' && strncmp(azArg[0], "ar", n)==0 ){
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
+  if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){
     open_db(p, 0);
     rc = arDotCommand(p, azArg, nArg);
   }else
@@ -5973,11 +6074,18 @@ static int do_meta_command(char *zLine, ShellState *p){
     p->zDbFilename = 0;
     sqlite3_free(p->zFreeOnClose);
     p->zFreeOnClose = 0;
+    p->openMode = SHELL_OPEN_UNSPEC;
     /* Check for command-line arguments */
     for(iName=1; iName<nArg && azArg[iName][0]=='-'; iName++){
       const char *z = azArg[iName];
       if( optionMatch(z,"new") ){
         newFlag = 1;
+#ifdef SQLITE_HAVE_ZIP
+      }else if( optionMatch(z, "zip") ){
+        p->openMode = SHELL_OPEN_ZIPFILE;
+#endif
+      }else if( optionMatch(z, "append") ){
+        p->openMode = SHELL_OPEN_APPENDVFS;
       }else if( z[0]=='-' ){
         utf8_printf(stderr, "unknown option: %s\n", z);
         rc = 1;
@@ -7846,6 +7954,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
         utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]);
         exit(1);
       }
+#ifdef SQLITE_HAVE_ZIP
+    }else if( strcmp(z,"-zip")==0 ){
+      data.openMode = SHELL_OPEN_ZIPFILE;
+#endif
+    }else if( strcmp(z,"-append")==0 ){
+      data.openMode = SHELL_OPEN_APPENDVFS;
     }
   }
   if( data.zDbFilename==0 ){
@@ -7858,6 +7972,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #endif
   }
   data.out = stdout;
+  sqlite3_appendvfs_init(0,0,0);
 
   /* Go ahead and open the database file if it already exists.  If the
   ** file does not exist, delay opening it.  This prevents empty database
@@ -7898,6 +8013,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( strcmp(z,"-csv")==0 ){
       data.mode = MODE_Csv;
       memcpy(data.colSeparator,",",2);
+#ifdef SQLITE_HAVE_ZIP
+    }else if( strcmp(z,"-zip")==0 ){
+      data.openMode = SHELL_OPEN_ZIPFILE;
+#endif
+    }else if( strcmp(z,"-append")==0 ){
+      data.openMode = SHELL_OPEN_APPENDVFS;
     }else if( strcmp(z,"-ascii")==0 ){
       data.mode = MODE_Ascii;
       sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,

src/test1.c

@@ -4559,6 +4559,35 @@ static int SQLITE_TCLAPI test_complete16(
   return TCL_OK;
 }
 
+/*
+** Usage: sqlite3_normalize SQL
+**
+** Return the normalized value for an SQL statement.
+*/
+static int SQLITE_TCLAPI test_normalize(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  char *zSql;
+  char *zNorm;
+  extern char *sqlite3_normalize(const char*);
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SQL");
+    return TCL_ERROR;
+  }
+
+  zSql = (char*)Tcl_GetString(objv[1]);
+  zNorm = sqlite3_normalize(zSql);
+  if( zNorm ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(zNorm, -1));
+    sqlite3_free(zNorm);
+  }
+  return TCL_OK;
+}
+
 /*
 ** Usage: sqlite3_step STMT
 **
@@ -7547,6 +7576,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_open16",                test_open16        ,0 },
      { "sqlite3_open_v2",               test_open_v2       ,0 },
      { "sqlite3_complete16",            test_complete16    ,0 },
+     { "sqlite3_normalize",             test_normalize     ,0 },
 
      { "sqlite3_prepare",               test_prepare       ,0 },
      { "sqlite3_prepare16",             test_prepare16     ,0 },

src/test_windirent.c

@@ -14,7 +14,6 @@
 */
 
 #if defined(_WIN32) && defined(_MSC_VER)
-
 #include "test_windirent.h"
 
 /*

src/test_windirent.h

@@ -13,7 +13,8 @@
 ** POSIX functions on Win32 using the MSVCRT.
 */
 
-#if defined(_WIN32) && defined(_MSC_VER)
+#if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H)
+#define SQLITE_WINDIRENT_H
 
 /*
 ** We need several data types from the Windows SDK header.

test/normalize.test

@@ -0,0 +1,72 @@
+# 2018-01-08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for the sqlite3_normalize() extension function.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix normalize
+
+foreach {tnum sql norm} {
+  100
+  {SELECT * FROM t1 WHERE a IN (1) AND b=51.42}
+  {select*from t1 where a in(?,?,?)and b=?;}
+
+  110
+  {SELECT a, b+15, c FROM t1 WHERE d NOT IN (SELECT x FROM t2);}
+  {select a,b+?,c from t1 where d not in(select x from t2);}
+
+  120
+  { SELECT NULL, b FROM t1 -- comment text
+     WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */
+                 SELECT a FROM t)
+        OR e='hello';
+  }
+  {select?,b from t1 where d in(with t(a)as(values(?))select a from t)or e=?;}
+
+  121
+  {/*Initial comment*/
+   -- another comment line
+   SELECT NULL  /* comment */ , b FROM t1 -- comment text
+     WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */
+                 SELECT a FROM t)
+        OR e='hello';
+  }
+  {select?,b from t1 where d in(with t(a)as(values(?))select a from t)or e=?;}
+
+  130
+  {/* Query containing parameters */
+   SELECT x,$::abc(15),y,@abc,z,?99,w FROM t1 /* Trailing comment */}
+  {select x,?,y,?,z,?,w from t1;}
+
+  140
+  {/* Long list on the RHS of IN */
+   SELECT 15 IN (1,2,3,(SELECT * FROM t1),'xyz',x'abcd',22*(x+5),null);}
+  {select?in(?,?,?);}
+
+  150
+  {SELECT x'abc'; -- illegal token}
+  {}
+
+  160
+  {SELECT a,NULL,b FROM t1 WHERE c IS NOT NULL or D is null or e=5}
+  {select a,?,b from t1 where c is not null or d is null or e=?;}
+
+  170
+  {/* IN list exactly 5 bytes long */
+   SELECT * FROM t1 WHERE x IN (1,2,3);}
+  {select*from t1 where x in(?,?,?);}
+} {
+  do_test $tnum [list sqlite3_normalize $sql] $norm
+}
+
+finish_test

test/releasetest.tcl

@@ -734,6 +734,9 @@ proc makeCommand { targets makeOpts cflags opts } {
     set nmakeDir [file nativename $::SRCDIR]
     set nmakeFile [file nativename [file join $nmakeDir Makefile.msc]]
     lappend result nmake /f $nmakeFile TOP=$nmakeDir
+    set tclDir [file nativename [file normalize \
+        [file dirname [file dirname [info nameofexecutable]]]]]
+    lappend result "TCLDIR=$tclDir"
     if {[regexp {USE_STDCALL=1} $cflags]} {
       lappend result USE_STDCALL=1
     }

test/zipfile.test

@@ -14,7 +14,10 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix zipfile
 
-load_static_extension db zipfile
+if {[catch {load_static_extension db zipfile} error]} {
+  puts "Skipping zipfile tests, hit load error: $error"
+  finish_test; return
+}
 
 forcedelete test.zip
 do_execsql_test 1.0 {

tool/mkshellc.tcl

@@ -30,16 +30,29 @@ puts $out {/* DO NOT EDIT!
 ** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script.
 */}
 set in [open $topdir/src/shell.c.in rb]
+proc omit_redundant_typedefs {line} {
+  global typedef_seen
+  if {[regexp {^typedef .*;} $line]} {
+    if {[info exists typedef_seen($line)]} {
+      return "/* $line */"
+    }
+    set typedef_seen($line) 1
+  }
+  return $line
+}
 while {1} {
-  set lx [gets $in]
+  set lx [omit_redundant_typedefs [gets $in]]
   if {[eof $in]} break;
   if {[regexp {^INCLUDE } $lx]} {
     set cfile [lindex $lx 1]
     puts $out "/************************* Begin $cfile ******************/"
     set in2 [open $topdir/src/$cfile rb]
     while {![eof $in2]} {
-      set lx [gets $in2]
+      set lx [omit_redundant_typedefs [gets $in2]]
       if {[regexp {^#include "sqlite} $lx]} continue
+      if {[regexp {^# *include "test_windirent.h"} $lx]} {
+        set lx "/* $lx */"
+      }
       set lx [string map [list __declspec(dllexport) {}] $lx]
       puts $out $lx
     }

tool/sqltclsh.c.in

@@ -0,0 +1,47 @@
+/*
+** This is the source code to a "tclsh" that has SQLite built-in.
+**
+** The startup script is located as follows:
+**
+**   (1)  Open the executable as an appended SQLite database and try to
+**        read the startup script out of that database.
+**
+**   (2)  If the first argument is a readable file, try to open that file
+**        as an SQLite database and read the startup script out of that
+**        database.
+**
+**   (3)  If the first argument is a readable file with a ".tcl" extension,
+**        then try to run that script directly.
+**
+** If none of the above steps work, then the program runs as an interactive
+** tclsh.
+*/
+#define TCLSH_INIT_PROC sqlite3_tclapp_init_proc
+#define SQLITE_ENABLE_DBSTAT_VTAB 1
+#undef SQLITE_THREADSAFE
+#define SQLITE_THREADSAFE 0
+#undef SQLITE_ENABLE_COLUMN_METADATA
+#define SQLITE_OMIT_DECLTYPE 1
+#define SQLITE_OMIT_DEPRECATED 1
+#define SQLITE_OMIT_PROGRESS_CALLBACK 1
+#define SQLITE_OMIT_SHARED_CACHE 1
+#define SQLITE_DEFAULT_MEMSTATUS 0
+#define SQLITE_MAX_EXPR_DEPTH 0
+INCLUDE sqlite3.c
+INCLUDE $ROOT/ext/misc/appendvfs.c
+INCLUDE $ROOT/ext/misc/zipfile.c
+INCLUDE $ROOT/ext/misc/sqlar.c
+INCLUDE $ROOT/src/tclsqlite.c
+
+const char *sqlite3_tclapp_init_proc(Tcl_Interp *interp){
+  (void)interp;
+  sqlite3_appendvfs_init(0,0,0);
+  sqlite3_auto_extension((void(*)(void))sqlite3_sqlar_init);
+  sqlite3_auto_extension((void(*)(void))sqlite3_zipfile_init);
+
+  return
+BEGIN_STRING
+INCLUDE $ROOT/tool/sqltclsh.tcl
+END_STRING
+;
+}

tool/sqltclsh.tcl

@@ -0,0 +1,71 @@
+# Try to open the executable as a database and read the "scripts.data"
+# field where "scripts.name" is 'main.tcl'
+#
+catch {
+  if {![file exists $argv0] && [file exists $argv0.exe]} {
+    append argv0 .exe
+  }
+  sqlite3 db $argv0 -vfs apndvfs -create 0
+  set mainscript [db one {
+      SELECT sqlar_uncompress(data,sz) FROM sqlar WHERE name='main.tcl'
+  }]
+}
+if {[info exists mainscript]} {
+  eval $mainscript
+  return
+} else {
+  catch {db close}
+}
+
+# Try to open file named in the first argument as a database and
+# read the "scripts.data" field where "scripts.name" is 'main.tcl'
+#
+if {[llength $argv]>0 && [file readable [lindex $argv 0]]} {
+  catch {
+    sqlite3 db [lindex $argv 0] -vfs apndvfs -create 0
+    set mainscript [db one {SELECT data FROM scripts WHERE name='main.tcl'}]
+    set argv0 [lindex $argv 0]
+    set argv [lrange $argv 1 end]
+  }
+  if {[info exists mainscript]} {
+    eval $mainscript
+    return
+  } else {
+    catch {db close}
+  }
+  if {[string match *.tcl [lindex $argv 0]]} {
+    set fd [open [lindex $argv 0] rb]
+    set mainscript [read $fd]
+    close $fd
+    unset fd
+    set argv0 [lindex $argv 0]
+    set argv [lrange $argv 1 end]
+  }
+  if {[info exists mainscript]} {
+    eval $mainscript
+    return
+  }
+}
+
+# If all else fails, do an interactive loop
+#
+set line {}
+while {![eof stdin]} {
+  if {$line!=""} {
+    puts -nonewline "> "
+  } else {
+    puts -nonewline "% "
+  }
+  flush stdout
+  append line [gets stdin]
+  if {[info complete $line]} {
+    if {[catch {uplevel #0 $line} result]} {
+      puts stderr "Error: $result"
+    } elseif {$result!=""} {
+      puts $result
+    }
+    set line {}
+  } else {
+    append line \\n"
+  }
+}