Update zonefile README.md file to mention the frame cache.

FossilOrigin-Name: 84e9351bbd0a9bf9e1d64a441366a90276c231b44548cbc947f7fb2842f82490
Rationalize some code in zonefile.c. Fix other minor issues in the same.
2018-02-27 20:09:45 +00:00 · 2018-02-27 19:50:28 +00:00 · 2018-02-27 15:47:41 +00:00 · 2018-02-27 15:42:29 +00:00 · 2018-02-27 14:26:33 +00:00 · 2018-02-26 07:58:39 +00:00
38 changed files with 4187 additions and 688 deletions
@@ -591,6 +591,7 @@ SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
 DBFUZZ_OPT = 

@@ -1494,6 +1494,7 @@ TESTEXT = \
  $(TOP)\ext\misc\eval.c \
  $(TOP)\ext\misc\fileio.c \
  $(TOP)\ext\misc\fuzzer.c \
+  $(TOP)\ext\zonefile\zonefile.c \
  $(TOP)\ext\fts5\fts5_tcl.c \
  $(TOP)\ext\fts5\fts5_test_mi.c \
  $(TOP)\ext\fts5\fts5_test_tok.c \
@@ -1615,7 +1616,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_RTREE
 #
 MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
 OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
 DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
@@ -2252,6 +2253,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)

 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
@@ -14,15 +14,17 @@ SQLite sources are managed using the
 [Fossil](https://www.fossil-scm.org/), a distributed version control system
 that was specifically designed to support SQLite development.
 If you do not want to use Fossil, you can download tarballs or ZIP
-archives as follows:
+archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:

-  *  Lastest trunk check-in:
-     <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
-     <https://www.sqlite.org/src/zip/sqlite.zip>.
+  *  Lastest trunk check-in as
+     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz),
+     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or
+     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar).

-  *  Latest release:
-     <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
-     <https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
+  *  Latest release as
+     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release),
+     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or
+     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release).

  *  For other check-ins, substitute an appropriate branch name or
     tag or hash prefix for "release" in the URLs of the previous
@@ -5529,7 +5529,6 @@ int sqlite3Fts3UpdateMethod(
 ){
  Fts3Table *p = (Fts3Table *)pVtab;
  int rc = SQLITE_OK;             /* Return Code */
-  int isRemove = 0;               /* True for an UPDATE or DELETE */
  u32 *aSzIns = 0;                /* Sizes of inserted documents */
  u32 *aSzDel = 0;                /* Sizes of deleted documents */
  int nChng = 0;                  /* Net change in number of documents */
@@ -5627,7 +5626,6 @@ int sqlite3Fts3UpdateMethod(
  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
    rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
-    isRemove = 1;
  }
  
  /* If this is an INSERT or UPDATE operation, insert the new record. */
@@ -5639,7 +5637,7 @@ int sqlite3Fts3UpdateMethod(
        rc = FTS_CORRUPT_VTAB;
      }
    }
-    if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
+    if( rc==SQLITE_OK ){
      rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
    }
    if( rc==SQLITE_OK ){
@@ -18,6 +18,12 @@
 SQLITE_EXTENSION_INIT1

 #ifndef SQLITE_AMALGAMATION
+# if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
+#  define NDEBUG 1
+# endif
+# if defined(NDEBUG) && defined(SQLITE_DEBUG)
+#  undef NDEBUG
+# endif
 # include <string.h>
 # include <stdio.h>
 # include <stdlib.h>
@@ -652,6 +658,79 @@ static void editDist3ConfigDelete(void *pIn){
  sqlite3_free(p);
 }

+/* Compare the FROM values of two EditDist3Cost objects, for sorting.
+** Return negative, zero, or positive if the A is less than, equal to,
+** or greater than B.
+*/
+static int editDist3CostCompare(EditDist3Cost *pA, EditDist3Cost *pB){
+  int n = pA->nFrom;
+  int rc;
+  if( n>pB->nFrom ) n = pB->nFrom;
+  rc = strncmp(pA->a, pB->a, n);
+  if( rc==0 ) rc = pA->nFrom - pB->nFrom;
+  return rc;
+}
+
+/*
+** Merge together two sorted lists of EditDist3Cost objects, in order
+** of increasing FROM.
+*/
+static EditDist3Cost *editDist3CostMerge(
+  EditDist3Cost *pA,
+  EditDist3Cost *pB
+){
+  EditDist3Cost *pHead = 0;
+  EditDist3Cost **ppTail = &pHead;
+  EditDist3Cost *p;
+  while( pA && pB ){
+    if( editDist3CostCompare(pA,pB)<=0 ){
+      p = pA;
+      pA = pA->pNext;
+    }else{
+      p = pB;
+      pB = pB->pNext;
+    }
+    *ppTail = p;
+    ppTail =  &p->pNext;
+  }
+  if( pA ){
+    *ppTail = pA;
+  }else{
+    *ppTail = pB;
+  }
+  return pHead;
+}
+
+/*
+** Sort a list of EditDist3Cost objects into order of increasing FROM
+*/
+static EditDist3Cost *editDist3CostSort(EditDist3Cost *pList){
+  EditDist3Cost *ap[60], *p;
+  int i;
+  int mx = 0;
+  ap[0] = 0;
+  ap[1] = 0;
+  while( pList ){
+    p = pList;
+    pList = p->pNext;
+    p->pNext = 0;
+    for(i=0; ap[i]; i++){
+      p = editDist3CostMerge(ap[i],p);
+      ap[i] = 0;
+    }
+    ap[i] = p;
+    if( i>mx ){
+      mx = i;
+      ap[i+1] = 0;
+    }
+  }
+  p = 0;
+  for(i=0; i<=mx; i++){
+    if( ap[i] ) p = editDist3CostMerge(p,ap[i]);
+  }
+  return p;
+}
+
 /*
 ** Load all edit-distance weights from a table.
 */
@@ -685,6 +764,7 @@ static int editDist3ConfigLoad(
    assert( zTo!=0 || nTo==0 );
    if( nFrom>100 || nTo>100 ) continue;
    if( iCost<0 ) continue;
+    if( iCost>10000 ) continue;   /* Costs above 10K are considered infinite */
    if( pLang==0 || iLang!=iLangPrev ){
      EditDist3Lang *pNew;
      pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0]));
@@ -722,6 +802,12 @@ static int editDist3ConfigLoad(
  }
  rc2 = sqlite3_finalize(pStmt);
  if( rc==SQLITE_OK ) rc = rc2;
+  if( rc==SQLITE_OK ){
+    int iLang;
+    for(iLang=0; iLang<p->nLang; iLang++){
+      p->a[iLang].pCost = editDist3CostSort(p->a[iLang].pCost);
+    }
+  }
  return rc;
 }

@@ -749,6 +835,7 @@ static int utf8Len(unsigned char c, int N){
 ** the given string.
 */
 static int matchTo(EditDist3Cost *p, const char *z, int n){
+  if( p->a[p->nFrom]!=z[0] ) return 0;
  if( p->nTo>n ) return 0;
  if( strncmp(p->a+p->nFrom, z, p->nTo)!=0 ) return 0;
  return 1;
@@ -760,6 +847,7 @@ static int matchTo(EditDist3Cost *p, const char *z, int n){
 */
 static int matchFrom(EditDist3Cost *p, const char *z, int n){
  assert( p->nFrom<=n );
+  if( p->a[0]!=z[0] ) return 0;
  if( strncmp(p->a, z, p->nFrom)!=0 ) return 0;
  return 1;
 }
@@ -776,7 +864,8 @@ static int matchFromTo(
 ){
  int b1 = pStr->a[n1].nByte;
  if( b1>n2 ) return 0;
-  if( memcmp(pStr->z+n1, z2, b1)!=0 ) return 0;
+  if( pStr->z[n1]!=z2[0] ) return 0;
+  if( strncmp(pStr->z+n1, z2, b1)!=0 ) return 0;
  return 1;
 }

@@ -858,9 +947,6 @@ static EditDist3FromString *editDist3FromStringNew(
 /*
 ** Update entry m[i] such that it is the minimum of its current value
 ** and m[j]+iCost.
-**
-** If the iCost is 1,000,000 or greater, then consider the cost to be
-** infinite and skip the update.
 */
 static void updateCost(
  unsigned int *m,
@@ -868,11 +954,11 @@ static void updateCost(
  int j,
  int iCost
 ){
+  unsigned int b;
  assert( iCost>=0 );
-  if( iCost<10000 ){
-    unsigned int b = m[j] + iCost;
-    if( b<m[i] ) m[i] = b;
-  }
+  assert( iCost<10000 );
+  b = m[j] + iCost;
+  if( b<m[i] ) m[i] = b;
 }

 /*
@@ -936,8 +1022,9 @@ static int editDist3Core(
    a2[i2].nByte = utf8Len((unsigned char)z2[i2], n2-i2);
    for(p=pLang->pCost; p; p=p->pNext){
      EditDist3Cost **apNew;
-      if( p->nFrom>0 ) continue;
+      if( p->nFrom>0 ) break;
      if( i2+p->nTo>n2 ) continue;
+      if( p->a[0]>z2[i2] ) break;
      if( matchTo(p, z2+i2, n2-i2)==0 ) continue;
      a2[i2].nIns++;
      apNew = sqlite3_realloc64(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns);
@@ -2492,7 +2579,7 @@ static int spellfix1FilterForMatch(
  nPattern = (int)strlen(zPattern);
  if( zPattern[nPattern-1]=='*' ) nPattern--;
  zSql = sqlite3_mprintf(
-     "SELECT id, word, rank, k1"
+     "SELECT id, word, rank, coalesce(k1,word)"
     "  FROM \"%w\".\"%w_vocab\""
     " WHERE langid=%d AND k2>=?1 AND k2<?2",
     p->zDbName, p->zTableName, iLang
@@ -2826,17 +2913,17 @@ static int spellfix1Update(
      if( sqlite3_value_type(argv[1])==SQLITE_NULL ){
        spellfix1DbExec(&rc, db,
               "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) "
-               "VALUES(%d,%d,%Q,%Q,%Q)",
+               "VALUES(%d,%d,%Q,nullif(%Q,%Q),%Q)",
               p->zDbName, p->zTableName,
-               iRank, iLang, zWord, zK1, zK2
+               iRank, iLang, zWord, zK1, zWord, zK2
        );
      }else{
        newRowid = sqlite3_value_int64(argv[1]);
        spellfix1DbExec(&rc, db,
            "INSERT OR %s INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) "
-            "VALUES(%lld,%d,%d,%Q,%Q,%Q)",
+            "VALUES(%lld,%d,%d,%Q,nullif(%Q,%Q),%Q)",
            zConflict, p->zDbName, p->zTableName,
-            newRowid, iRank, iLang, zWord, zK1, zK2
+            newRowid, iRank, iLang, zWord, zK1, zWord, zK2
        );
      }
      *pRowid = sqlite3_last_insert_rowid(db);
@@ -2845,9 +2932,9 @@ static int spellfix1Update(
      newRowid = *pRowid = sqlite3_value_int64(argv[1]);
      spellfix1DbExec(&rc, db,
             "UPDATE OR %s \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d,"
-             " word=%Q, k1=%Q, k2=%Q WHERE id=%lld",
+             " word=%Q, k1=nullif(%Q,%Q), k2=%Q WHERE id=%lld",
             zConflict, p->zDbName, p->zTableName, newRowid, iRank, iLang,
-             zWord, zK1, zK2, rowid
+             zWord, zK1, zWord, zK2, rowid
      );
    }
    sqlite3_free(zK1);
@@ -737,7 +737,7 @@ static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){
  int hr, min, sec;

  A = (int)((JD - 1867216.25)/36524.25);
-  A = JD + 1 + A - (A/4);
+  A = (int)(JD + 1 + A - (A/4));
  B = A + 1524;
  C = (int)((B - 122.1)/365.25);
  D = (36525*(C&32767))/100;
@@ -1623,7 +1623,7 @@ static int zipfileUpdate(
        pNew->cds.szUncompressed = (u32)sz;
        pNew->cds.iExternalAttr = (mode<<16);
        pNew->cds.iOffset = (u32)pTab->szCurrent;
-        pNew->cds.nFile = nPath;
+        pNew->cds.nFile = (u16)nPath;
        pNew->mUnixTime = (u32)mTime;
        rc = zipfileAppendEntry(pTab, pNew, pData, nData);
        zipfileAddEntry(pTab, pOld, pNew);
@@ -1926,7 +1926,7 @@ void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
  /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use
  ** deflate compression) or NULL (choose automatically).  */
  if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){
-    iMethod = sqlite3_value_int64(pMethod);
+    iMethod = (int)sqlite3_value_int64(pMethod);
    if( iMethod!=0 && iMethod!=8 ){
      zErr = sqlite3_mprintf("illegal method value: %d", iMethod);
      rc = SQLITE_ERROR;
@@ -1993,14 +1993,14 @@ void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
  e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
  e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
  e.cds.flags = ZIPFILE_NEWENTRY_FLAGS;
-  e.cds.iCompression = iMethod;
+  e.cds.iCompression = (u16)iMethod;
  zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime);
  e.cds.crc32 = iCrc32;
  e.cds.szCompressed = nData;
  e.cds.szUncompressed = szUncompressed;
  e.cds.iExternalAttr = (mode<<16);
  e.cds.iOffset = p->body.n;
-  e.cds.nFile = nName;
+  e.cds.nFile = (u16)nName;
  e.cds.zFile = zName;

  /* Append the LFH to the body of the new archive */
@@ -2047,8 +2047,8 @@ void zipfileFinal(sqlite3_context *pCtx){
  if( p==0 ) return;
  if( p->nEntry>0 ){
    memset(&eocd, 0, sizeof(eocd));
-    eocd.nEntry = p->nEntry;
-    eocd.nEntryTotal = p->nEntry;
+    eocd.nEntry = (u16)p->nEntry;
+    eocd.nEntryTotal = (u16)p->nEntry;
    eocd.nSize = p->cds.n;
    eocd.iOffset = p->body.n;

@@ -30,35 +30,80 @@ is the name of the database table to read and optional argument
 &lt;parameters&gt; is a JSON object containing various attributes that
 influence creation of the zonefile file. 

-Currently the only &lt;parameters&gt; attribute supported is 
-<i>maxAutoFrameSize</i> (default value 65536), which sets the maximum 
-uncompressed frame size in bytes for automatically generated zonefile 
-frames.
+Currently, the following &lt;parameters&gt; attributes are supported:
+
+<table border=1>
+<tr align=left><th>Attribute<th>Default<th>Interpretation
+<tr valign=top><td>maxAutoFrameSize<td>65536
+<td>The maximum uncompressed frame size in bytes for automatically generated
+zonefile frames.
+
+<tr valign=top><td>compressionTypeContent<td>"none"
+<td>The compression type used to compress each frame in the zonefile. 
+Valid values are "none" (no compression), "zstd", "zstd_global_dict",
+"zlib", "brotli", "lz4" and "lz4hc". Not all compression methods are
+supported by all builds. The compression method supported by a build
+depends on the combination of SQLITE_HAVE_ZSTD, SQLITE_HAVE_ZLIB,
+SQLITE_HAVE_BROTLI and SQLITE_HAVE_LZ4 pre-processor symbols defined
+at build time.
+
+<tr valign=top><td>compressionTypeIndexData<td>"none"
+<td>The compression type used to compress the zonefile index structure.
+All values that are valid for the <i>compressionTypeContent</i> parameter,
+except for "zstd_global_dict", are also valid for this option.
+
+<tr valign=top><td>encryptionType<td>"none"
+<td>The encryption type to use. At present the only valid values are
+"none" (no encryption) and "xor" (an insecure mock encryption method
+useful for testing only). Enhanced implementations may support any or
+all of the following encryption schemes:
+<ul>
+  <li> "AES_128_CTR"
+  <li> "AES_128_CBC"
+  <li> "AES_256_CTR"
+  <li> "AES_256_CBC"
+</ul>
+
+<tr valign=top><td>encryptionKey<td>""
+<td>The encryption key to use. The encryption key must be specified as an
+even number of hexadecimal that will be converted to a binary key before
+use. It is the responsibility of the caller to specify a key of the optimal
+length for each encryption algorithm (e.g. 16 bytes (32 hex digits) for
+a 128-bit encryption, or 32 bytes (64 digits) for a 256-bit method).
+This option is ignored if <i>encryptionType</i> is set to "none".
+</table>

 For example, to create a zonefile named "test.zonefile" based on the
-contents of database table "test_input" and with a maximum automatic
-frame size of 4096 bytes:
+contents of database table "test_input", with a maximum automatic
+frame size of 4096 bytes and using "xor" encryption with a 128-bit key:

 >     SELECT zonefile_write('test.zonefile', 'test_input',
->       '{"maxAutoFrameSize":4096}'
+>       '{"maxAutoFrameSize":4096,
+>         "encryptionType":"xor",
+>         "encryptionKey":"e6e600bc063aad12f6387beab650c48a"
+>       }'
 >     );

 ### Using (Reading) Zonefile Files

-To create a new zonefile table:
+To create a new zonefile table, one of the following:

 >     CREATE VIRTUAL TABLE z1 USING zonefile;
+>     CREATE VIRTUAL TABLE z1 USING zonefile(cachesize=N);

-This creates two virtual tables in the database schema. One read-only table
-named "z1", with a schema equivalent to:
+where <i>N</i> is any non-zero positive integer. If the zonefile is used
+to access any files containing compressed or encrypted data, it maintains
+an LRU cache of uncompressed frame data <i>N</i> frames in size. The
+default value of <i>N</i> is 1.
+
+Creating a "zonefile" virtual table actually creates two virtual tables in the
+database schema. One read-only table named "z1", with a schema equivalent to:

 >     CREATE TABLE z1(  -- this whole table is read-only
->       k INTEGER PRIMARY KEY,
->       v BLOB,
->       fileid INTEGER,
->       frame INTEGER,
->       ofst INTEGER,
->       sz INTEGER
+>       k INTEGER PRIMARY KEY,     -- key value
+>       v BLOB,                    -- associated blob of data
+>       fileid INTEGER,            -- file id (rowid value for z1_files)
+>       sz INTEGER                 -- size of blob of data in bytes
 >     );

 And a read-write table named "z1_files" with a schema like:
@@ -66,7 +111,6 @@ And a read-write table named "z1_files" with a schema like:
 >     CREATE TABLE z1_files(
 >       filename TEXT PRIMARY KEY,
 >       ekey BLOB,         -- encryption key
->       fileid INTEGER,    -- read-only
 >       header JSON HIDDEN -- read-only
 >     );

@@ -75,9 +119,21 @@ row into the "z1_files" table:

 >     INSERT INTO z1_files(filename) VALUES(<filename>);

-Currently, any value provided for any column other than "filename" is 
-ignored. Files are removed from the index by deleting rows from the
-z1_files table:
+If the file is an encrypted file, then the encryption key (a blob) must
+be inserted into the "ekey" column. Encryption keys are not stored in the
+database, they are held in main-memory only. This means that each new
+connection must configure encryption key using UPDATE statements before
+accessing any encrypted files. For example:
+
+>     -- Add new encrypted file to database:
+>     INSERT INTO z1_files(filename, ekey) VALUES(<filename>, <ekey>);
+>
+>     -- Configure encryption key for existing file after opening database:
+>     UPDATE z1_files SET ekey = <ekey> WHERE filename = <filename>;
+
+Currently, values provided for any columns other than "filename" and
+"ekey" are ignored. Files are removed from the index by deleting rows 
+from the z1_files table:

 >     DELETE FROM z1_files WHERE filename = <filename>;

@@ -111,7 +167,17 @@ key from one of the zonefile files in the index:
     necessary as we may not know the offset of the start of the frame data
     until after the ZoneFileIndex structure is compressed.

-  *  Currently there is no support at all for encryption or compression.
+  *  The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are the
+     offsets for the first byte past the end of the corresponding frame.
+     For example, byteOffsetZoneFrame[] identifies the first byte of the
+     second frame, and byteOffsetZoneFrame[numFrames-1] is one byte past
+     the end of the last frame in the file.
+
+     This is better as if we store the starting offset of each frame, there
+     is no way to determine the size of the last frame in the file without
+     trusting the filesize itself.
+
+  *  Currently there is no support at all for encryption.

  *  Zonefile currently uses json1 to parse the json argument to
     zonefile\_write(). And so must be used with an SQLITE\_ENABLE\_JSON1
@@ -48,7 +48,6 @@ do_execsql_test 1.4 { SELECT count(*) FROM z1_shadow_idx } 3

 do_execsql_test 1.5.1 { SELECT k FROM z1 } {1 2 3}
 do_execsql_test 1.5.2 { SELECT fileid FROM z1 } {1 1 1}
-do_execsql_test 1.5.3 { SELECT frame FROM z1 } {0 0 0}
 do_execsql_test 1.5.4 { SELECT sz FROM z1 } {100 100 100}

 do_execsql_test 1.5.5 {
@@ -64,89 +63,582 @@ do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0

 do_execsql_test 1.7 { DROP TABLE z1 }

-#-------------------------------------------------------------------------
+do_execsql_test 1.8 {
+  SELECT * FROM sqlite_master WHERE name LIKE 'z1%';
+}

+#-------------------------------------------------------------------------
+# Figure out which compression algorithms, if any, are supported by
+# this build. Populate the global list $COMPRESSION_METHODS with the
+# result.
 reset_db
 load_static_extension db zonefile
-
 do_execsql_test 2.0 {
-  CREATE TABLE zz(
+  CREATE TABLE bb(
    k INTEGER PRIMARY KEY, 
    frame INTEGER DEFAULT -1, 
    idx INTEGER DEFAULT -1, 
    v BLOB
  );
-  CREATE TABLE rt(k INTEGER PRIMARY KEY, v BLOB);
-  CREATE VIRTUAL TABLE zone USING zonefile;
+  INSERT INTO bb(k, v) VALUES(1, randomblob(100));
 }
-
-set nMinByte   0
-set nMaxByte 444
-foreach {zonefile lKey} {
-  test1.zonefile {195 1238 298 405 297}
-  test2.zonefile {124 1624 82 1929}
-  test3.zonefile {932 683 1751 410 41}
-  test4.zonefile {427 1491}
-  test5.zonefile {1004 473 801 394 1672 816 1577}
-  test6.zonefile {1374 1454 1005}
-  test7.zonefile {450 241 319 133}
-  test8.zonefile {1414 900 1406 1917 127 673}
-  test9.zonefile {1192 226 988 1292 718 1345 1675}
-  test10.zonefile {314}
-  test11.zonefile {1177 1597 60 532 291 1164 812}
-  test12.zonefile {1168 1290 1585 939 1916}
-  test13.zonefile {644 1784 1476 1283 433 506}
-  test14.zonefile {1141 1547 1506 364}
-  test15.zonefile {1756 1885 844 1880 1896 354}
-  test16.zonefile {1383 1928 1371}
-  test17.zonefile {93}
-  test18.zonefile {1067}
-  test19.zonefile {642}
-  test20.zonefile {1380 1857}
-  test21.zonefile {288 293 1968 1207 1739 231 300}
-  test22.zonefile {651 1007 607 830 299 1431}
-  test23.zonefile {81 1651 543 1949 256 119 1088}
-  test24.zonefile {1278 2024 682 1115 194 636 1804}
-  test25.zonefile {514 1155 171 2015 791}
-  test26.zonefile {1615 1228 147 1464}
-  test27.zonefile {55 1130 781 678 78}
-  test28.zonefile {1981 1401 1178}
-  test29.zonefile {1754 864 183 1953 1901}
-  test30.zonefile {1461 817}
-  test31.zonefile {1720 1722 686 1833}
+set COMPRESSION_METHODS [list]
+foreach cmp {
+  none zlib zstd zstd_global_dict lz4 lz4hc brotli nosuchcmp
 } {
-  forcedelete $zonefile
-  execsql { DELETE FROM zz; }
-  foreach k $lKey {
-    execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
-  }
-  execsql { INSERT INTO rt SELECT k, v FROM zz }
-  breakpoint
-  execsql { 
-    SELECT zonefile_write($zonefile, 'zz', '{"maxAutoFrameSize":2000}');
-    INSERT INTO zone_files(filename) VALUES($zonefile);
+  set res [catchsql {
+    WITH p(n,v) AS (
+        VALUES('compressionTypeContent', $cmp)
+    )
+    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
+  }]
+
+  if {[lindex $res 0]==0} {
+    lappend COMPRESSION_METHODS $cmp
  }
 }

-do_execsql_test 2.1 {
-  SELECT k FROM zone JOIN rt USING (k) WHERE zone.v!=rt.v
-}
-do_execsql_test 2.2 {
-  SELECT count(*) FROM zone JOIN rt USING (k);
-} {135}
-do_execsql_test 2.3 {
-  SELECT filename, 
-         json_extract(header, '$.numKeys'),
-         json_extract(header, '$.numFrames')
-  FROM zone_files 
-  WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile')
-  ORDER BY 1
-} {
-  test19.zonefile 1 1
-  test20.zonefile 2 2
-  test21.zonefile 7 4
+# Check that it is not possible to use zstd_global_dict to compress
+# the zonefile index.
+#
+if {[lsearch $COMPRESSION_METHODS zstd_global_dict]>=0} {
+  do_catchsql_test 2.1 {
+    WITH p(n,v) AS (
+        VALUES('compressionTypeIndexData', 'zstd_global_dict')
+    )
+    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
+  } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
 }

+set extra_header 0
+set cachesize 0
+foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS {
+  if {$cmpidx == "zstd_global_dict"} continue
+  reset_db
+  load_static_extension db zonefile
+
+  set tn "$cmp/$cmpidx"
+  set extra_header [expr {$extra_header ? 0 : 100}]
+  set cachesize [expr {$cachesize ? 0 : 10}]
+
+  do_execsql_test 2.$tn.0.1 {
+    CREATE TABLE zz(
+      k INTEGER PRIMARY KEY, 
+      frame INTEGER DEFAULT -1, 
+      idx INTEGER DEFAULT -1, 
+      v BLOB
+    );
+    CREATE TABLE rt(k INTEGER PRIMARY KEY, v BLOB);
+  }
+
+  do_execsql_test 2.$tn.0.2 "
+    CREATE VIRTUAL TABLE zone USING zonefile(cachesize = $cachesize)
+  " {}
+  
+  set nMinByte   0
+  set nMaxByte 444
+  foreach {zonefile lKey} {
+    test1.zonefile {195 1238 298 405 297}
+    test2.zonefile {124 1624 82 1929}
+    test3.zonefile {932 683 1751 410 41}
+    test4.zonefile {427 1491}
+    test5.zonefile {1004 473 801 394 1672 816 1577}
+    test6.zonefile {1374 1454 1005}
+    test7.zonefile {450 241 319 133}
+    test8.zonefile {1414 900 1406 1917 127 673}
+    test9.zonefile {1192 226 988 1292 718 1345 1675}
+    test10.zonefile {314}
+    test11.zonefile {1177 1597 60 532 291 1164 812}
+    test12.zonefile {1168 1290 1585 939 1916}
+    test13.zonefile {644 1784 1476 1283 433 506}
+    test14.zonefile {1141 1547 1506 364}
+    test15.zonefile {1756 1885 844 1880 1896 354}
+    test16.zonefile {1383 1928 1371}
+    test17.zonefile {93}
+    test18.zonefile {1067}
+    test19.zonefile {642}
+    test20.zonefile {1380 1857}
+    test21.zonefile {288 293 1968 1207 1739 231 300}
+    test22.zonefile {651 1007 607 830 299 1431}
+    test23.zonefile {81 1651 543 1949 256 119 1088}
+    test24.zonefile {1278 2024 682 1115 194 636 1804}
+    test25.zonefile {514 1155 171 2015 791}
+    test26.zonefile {1615 1228 147 1464}
+    test27.zonefile {55 1130 781 678 78}
+    test28.zonefile {1981 1401 1178}
+    test29.zonefile {1754 864 183 1953 1901}
+    test30.zonefile {1461 817}
+    test31.zonefile {1720 1722 686 1833}
+  } {
+    forcedelete $zonefile
+    execsql { DELETE FROM zz; }
+    foreach k $lKey {
+      execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
+    }
+    execsql { INSERT INTO rt SELECT k, v FROM zz }
+    execsql { 
+      WITH p(n,v) AS (
+        VALUES('maxAutoFrameSize', 2000) UNION ALL
+        VALUES('compressionTypeIndexData', $cmpidx) UNION ALL
+        VALUES('compressionTypeContent', $cmp) UNION ALL
+        VALUES('debugExtendedHeaderSize', $extra_header)
+      )
+      SELECT zonefile_write($zonefile, 'zz', json_group_object(n, v)) FROM p;
+      INSERT INTO zone_files(filename) VALUES($zonefile);
+    }
+  }
+  
+  do_execsql_test 2.$tn.1 {
+    SELECT k FROM zone JOIN rt USING (k) WHERE zone.v!=rt.v
+  }
+  do_execsql_test 2.$tn.2 {
+    SELECT count(*) FROM zone JOIN rt USING (k);
+  } {135}
+  do_execsql_test 2.$tn.3 {
+    SELECT filename, 
+           json_extract(header, '$.numKeys'),
+           json_extract(header, '$.numFrames')
+    FROM zone_files 
+    WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile')
+    ORDER BY 1
+  } {
+    test19.zonefile 1 1
+    test20.zonefile 2 2
+    test21.zonefile 7 4
+  }
+}}
+
+#--------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 3.0 {
+  CREATE TABLE dd(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO dd VALUES(1000, 1, -1, randomblob(44));
+  INSERT INTO dd VALUES(1001, 1, -1, randomblob(55));
+  INSERT INTO dd VALUES(1002, 2, -1, randomblob(66));
+  WITH p(n,v) AS (
+      VALUES('maxAutoFrameSize', 2000) UNION ALL
+      VALUES('encryptionType', 'xor') UNION ALL
+      VALUES('debugEncryptionKeyText', 1) UNION ALL
+      VALUES('encryptionKey', '0123456789')
+  )
+  SELECT zonefile_write('test.zonefile', 'dd', json_group_object(n, v)) FROM p;
+} {{}}
+
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE cc USING zonefile;
+  INSERT INTO cc_files(filename,ekey) VALUES('test.zonefile','0123456789');
+  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
+} {1 1 1}
+
+do_execsql_test 3.2.1 {
+  DELETE FROM cc_files;
+  INSERT INTO cc_files(filename,ekey) VALUES('test.zonefile','abcdefghij');
+  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
+} {0 0 0}
+
+do_execsql_test 3.2.2 {
+  SELECT rowid,sz FROM cc;
+} {1000 44 1001 55 1002 66}
+
+do_execsql_test 3.3 {
+  UPDATE cc_files SET ekey = '0123456789';
+  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
+} {1 1 1}
+
+close [open test.zonefile w+]
+do_catchsql_test 3.4 {
+  SELECT header FROM cc_files
+} {1 {failed to read zonefile header from file "test.zonefile"}}
+
+forcedelete test.zonefile
+do_catchsql_test 3.5 {
+  SELECT header FROM cc_files
+} {1 {failed to open file "test.zonefile" for reading}}
+
+do_execsql_test 3.6 {
+  SELECT ekey FROM cc_files
+} {{}}
+
+forcedelete test.zonefile
+do_catchsql_test 3.7 {
+  SELECT * FROM cc;
+} {1 {failed to open file "test.zonefile" for reading}}
+
+#-------------------------------------------------------------------------
+# Check that a file that uses an unknown compression method is handled
+# correctly (an error is returned).
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 4.0 {
+  CREATE TABLE dd(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO dd VALUES(1000, 1, -1, randomblob(44));
+  INSERT INTO dd VALUES(1001, 1, -1, randomblob(55));
+  INSERT INTO dd VALUES(1002, 2, -1, randomblob(66));
+  SELECT zonefile_write('test.zonefile', 'dd');
+  CREATE VIRTUAL TABLE x1 USING zonefile;
+} {{}}
+
+do_test 4.1 {
+  hexio_write test.zonefile 5 77
+} {1}
+do_execsql_test 4.2 {
+  INSERT INTO x1_files(filename) VALUES('test.zonefile');
+} {}
+do_catchsql_test 4.3 {
+  SELECT * FROM x1
+} {1 {unsupported compression method: 119}}
+do_test 4.4 {
+  hexio_write test.zonefile 4 77
+} {1}
+do_catchsql_test 4.5 {
+  DELETE FROM x1_files;
+  INSERT INTO x1_files(filename) VALUES('test.zonefile');
+} {1 {unsupported compression method: 119}}
+
+do_test 4.6 {
+  hexio_write test.zonefile 0 00
+} {1}
+do_catchsql_test 4.7 {
+  INSERT INTO x1_files(filename) VALUES('test.zonefile');
+} {1 {failed to read zonefile header from file "test.zonefile"}}
+
+#-------------------------------------------------------------------------
+# Test using various types in the "frame" field of an input table.
+#
+reset_db
+load_static_extension db zonefile
+if {[lsearch $COMPRESSION_METHODS zlib]>=0} {
+  do_execsql_test 5.0 {
+    CREATE TABLE "a b"(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
+    INSERT INTO "a b" VALUES(1, 0.5, -1, randomblob(44));
+    INSERT INTO "a b" VALUES(2, 0.5, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(3, 1.5, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(4, 1.5, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(5,   2, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(6,   2, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(7, 200, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(8, 200, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(9, 300, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(10, 300, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(11, NULL, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(12, NULL, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(13, 'f1', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(14, 'f1', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(15, 'frame2', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(16, 'frame2', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(17, x'1234', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(18, x'1234', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(19, x'abcd', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(20, x'abcd', -1, randomblob(55));
+  
+    SELECT zonefile_write('test.zonefile', 'a b',
+      '{"compressionTypeContent":"zlib"}'
+    );
+  } {{}}
+
+  do_execsql_test 5.1 {
+    CREATE VIRTUAL TABLE abc USING zonefile; 
+    INSERT INTO abc_files(filename) VALUES('test.zonefile');
+    SELECT group_concat(k) FROM abc_shadow_idx GROUP BY fofst
+  } {
+    11,12   1,2   3,4   5,6   7,8  
+     9,10  13,14 15,16 17,18 19,20
+  }
+}
+
+do_execsql_test 6.0 {
+  CREATE TABLE "ab"(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
+  INSERT INTO "ab" VALUES(1, 0.5, -1, randomblob(44));
+  INSERT INTO "ab" VALUES(2, 0.5, -1, randomblob(55));
+  INSERT INTO "ab" VALUES(3, 1.5, -1, randomblob(55));
+  INSERT INTO "ab" VALUES(4, 1.5, -1, randomblob(55));
+}
+do_catchsql_test 6.1.1 {
+  SELECT zonefile_write('test.zonefile', 'ab',
+      '{"debugExtendedHeaderSize":-1}'
+  );
+} {1 {debugExtendedHeaderSize value out of range: -1}}
+do_catchsql_test 6.1.2 {
+  SELECT zonefile_write('test.zonefile', 'ab',
+      '{"debugExtendedHeaderSize":256}'
+  );
+} {1 {debugExtendedHeaderSize value out of range: 256}}
+
+do_catchsql_test 6.2 {
+  SELECT zonefile_write('test.zonefile', 'ab',
+      '{"unknown":256}'
+  );
+} {1 {unknown parameter name: "unknown"}}
+
+forcedelete test.dir
+file mkdir test.dir
+do_catchsql_test 6.3 {
+  SELECT zonefile_write('test.dir', 'ab');
+} {1 {failed to open file "test.dir" for writing}}
+
+do_catchsql_test 6.4 {
+  CREATE VIRTUAL TABLE zzz USING zonefile;
+  INSERT INTO zzz_files(filename) VALUES('nosuchfile.zonefile');
+} {1 {failed to open file "nosuchfile.zonefile" for reading}}
+
+if {$tcl_platform(platform)=="windows"} {
+  do_catchsql_test 6.5 {
+    INSERT INTO zzz_files(filename) VALUES('test.dir');
+  } {1 {failed to open file "test.dir" for reading}}
+} else {
+  do_catchsql_test 6.5 {
+    INSERT INTO zzz_files(filename) VALUES('test.dir');
+  } {1 {failed to read zonefile header from file "test.dir"}}
+}
+
+#-------------------------------------------------------------------------
+# Check that errors generated when building a dictionary are handled.
+# The zstd library routines for building a dictionary throw an error
+# if they are provided with too little data.
+#
+# Also test that zstd_global_dict cannot be used to compress the zonefile
+# index (as there is nowhere in the file format to store the dictionary
+# for this compression).
+#
+reset_db
+load_static_extension db zonefile
+if {[lsearch $COMPRESSION_METHODS zstd_global_dict]>=0} {
+  do_execsql_test 7.0 {
+    CREATE TABLE "ab"(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
+    INSERT INTO "ab" VALUES(1, -1, -1, 'abc');
+  }
+
+  do_catchsql_test 7.1 {
+    SELECT zonefile_write('test.zonefile', 'ab',
+      '{"compressionTypeContent":"zstd_global_dict"}'
+    );
+  } {1 {error generating dictionary}}
+
+  do_catchsql_test 7.2 {
+    SELECT zonefile_write('test.zonefile', 'ab',
+      '{"compressionTypeIndexData":"zstd_global_dict"}'
+    );
+  } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_catchsql_test 8.1 {
+  CREATE VIRTUAL TABLE one USING zonefile_files;
+} {1 {do not create zonefile_files tables directly!}}
+do_catchsql_test 8.2 {
+  CREATE VIRTUAL TABLE onetwothree USING zonefile_files;
+} {1 {do not create zonefile_files tables directly!}}
+
+#-------------------------------------------------------------------------
+# A zone file containing zero keys.
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 9.0 {
+  CREATE TABLE s(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
+  SELECT zonefile_write('test.zonefile', 's');
+} {{}}
+do_execsql_test 9.1 {
+  CREATE VIRTUAL TABLE ss USING zonefile;
+  INSERT INTO ss_files(filename) VALUES('test.zonefile');
+  SELECT * FROM ss;
+} {}
+
+#-------------------------------------------------------------------------
+# Test UPDATE and DELETE statements on the %_files virtual table.
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 10.0 {
+  CREATE TABLE data(k INTEGER PRIMARY KEY, v BLOB);
+  INSERT INTO data(k, v) VALUES(1, randomblob(100));
+  INSERT INTO data(k, v) VALUES(2, randomblob(100));
+  INSERT INTO data(k, v) VALUES(3, randomblob(100));
+  INSERT INTO data(k, v) VALUES(4, randomblob(100));
+  INSERT INTO data(k, v) VALUES(5, randomblob(100));
+  INSERT INTO data(k, v) VALUES(6, randomblob(100));
+  INSERT INTO data(k, v) VALUES(7, randomblob(100));
+  INSERT INTO data(k, v) VALUES(8, randomblob(100));
+  INSERT INTO data(k, v) VALUES(9, randomblob(100));
+
+  CREATE VIEW v1 AS 
+    SELECT k, -1 AS frame, -1 AS idx, v
+    FROM data WHERE k IN (1,2,3);
+  SELECT zonefile_write('test1.zonefile', 'v1');
+
+  CREATE VIEW v2 AS 
+    SELECT k, -1 AS frame, -1 AS idx, v
+    FROM data WHERE k IN (4,5,6);
+  SELECT zonefile_write('test2.zonefile', 'v2');
+
+  CREATE VIEW v3 AS 
+    SELECT k, -1 AS frame, -1 AS idx, v
+    FROM data WHERE k IN (7,8,9);
+  SELECT zonefile_write('test3.zonefile', 'v3');
+} {{} {} {}}
+
+do_execsql_test 10.1 {
+  CREATE VIRTUAL TABLE "a b" USING zonefile;
+  INSERT INTO "a b_files"(filename) VALUES('test1.zonefile');
+  SELECT k FROM "a b"
+} {1 2 3}
+
+do_execsql_test 10.2 {
+  UPDATE "a b_files" SET filename = 'test2.zonefile';
+  SELECT k FROM "a b"
+} {4 5 6}
+
+do_execsql_test 10.3 {
+  INSERT INTO "a b_files"(filename) VALUES('test1.zonefile');
+  INSERT INTO "a b_files"(filename) VALUES('test3.zonefile');
+  SELECT k FROM "a b"
+} {1 2 3 4 5 6 7 8 9}
+
+do_execsql_test 10.4 {
+  DELETE FROM "a b_files" WHERE filename = 'test2.zonefile';
+  SELECT k FROM "a b"
+} {1 2 3 7 8 9}
+
+do_execsql_test 10.5 {
+  DELETE FROM "a b_files" WHERE filename = 'test1.zonefile';
+  SELECT k FROM "a b"
+} {7 8 9}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 11.0 {
+  CREATE TABLE data(k INTEGER PRIMARY KEY, v BLOB);
+  INSERT INTO data(k, v) VALUES(-1, randomblob(100));
+  INSERT INTO data(k, v) VALUES(1000, randomblob(100));
+  INSERT INTO data(k, v) VALUES(-1000, randomblob(100));
+  INSERT INTO data(k, v) VALUES(9223372036854775807, randomblob(100));
+  INSERT INTO data(k, v) VALUES(-9223372036854775807, randomblob(100));
+  INSERT INTO data(k, v) VALUES(-9223372036854775808, randomblob(100));
+
+  CREATE VIEW v1 AS SELECT k, -1 AS frame, -1 AS idx, v FROM data;
+  SELECT zonefile_write('test1.zonefile', 'v1');
+} {{}}
+
+do_execsql_test 11.1 {
+  CREATE VIRTUAL TABLE one USING zonefile;
+  INSERT INTO "one_files"(filename) VALUES('test1.zonefile');
+  SELECT k FROM one
+} {-9223372036854775808 -9223372036854775807 -1000 -1 1000 9223372036854775807}
+
+do_execsql_test 11.2 {
+  SELECT count(*) FROM data JOIN one USING (k) WHERE one.v==data.v
+} 6
+
+#-------------------------------------------------------------------------
+#
+do_catchsql_test 12.1 {
+  CREATE VIRTUAL TABLE nm USING zonefile(abcd)
+} {1 {parse error in option: abcd}}
+do_catchsql_test 12.2 {
+  CREATE VIRTUAL TABLE nm USING zonefile(a=b)
+} {1 {parse error in option: a=b}}
+do_catchsql_test 12.3 {
+  CREATE VIRTUAL TABLE nm USING zonefile(cachesiza=b)
+} {1 {parse error in option: cachesiza=b}}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 13.0 {
+  CREATE TABLE data(k INTEGER PRIMARY KEY, v BLOB);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<250
+  )
+  INSERT INTO data SELECT i, randomblob(100) FROM s;
+  CREATE VIEW v1 AS SELECT k, -1 AS frame, -1 AS idx, v FROM data;
+  SELECT zonefile_write('test1.zonefile', 'v1');
+
+  CREATE VIRTUAL TABLE nm USING zonefile;
+  INSERT INTO nm_files(filename) VALUES('test1.zonefile');
+} {{}}
+
+foreach {tn cond} {
+  1   "k > 30"
+  2   "k >= 100"
+  3   "k <= 100"
+  4   "k < 55"
+  5   "k LIKE '1%'"
+  6   "k BETWEEN 10 AND 20"
+  7   "k > 100 AND k < 200"
+} {
+  do_execsql_test 13.1.$tn.1 [subst {
+    SELECT count(*) FROM nm WHERE $cond
+  }] [db one "SELECT count(*) FROM data WHERE $cond"]
+
+  do_execsql_test 13.1.$tn.2 [subst {
+    SELECT count(*) FROM nm WHERE $cond AND
+      v!=(SELECT v FROM data WHERE k=nm.k);
+  }] 0
+}
+
+close [open test1.zonefile w+]
+do_catchsql_test 13.2.1 {
+  SELECT * FROM nm WHERE k=24;
+} {1 {SQL logic error}}
+forcedelete test1.zonefile
+do_catchsql_test 13.2.2 {
+  SELECT * FROM nm WHERE k=24;
+} {1 {failed to open file "test1.zonefile" for reading}}
+
+do_catchsql_test 13.3.1 {
+  DELETE FROM nm_shadow_file;
+  SELECT * FROM nm WHERE k=24;
+} {1 {database disk image is malformed}}
+do_catchsql_test 13.3.2 {
+  DROP TABLE nm_shadow_file;
+  SELECT * FROM nm WHERE k=24;
+} {1 {no such table: main.nm_shadow_file}}
+db close
+sqlite3 db test.db
+load_static_extension db zonefile
+do_catchsql_test 13.3.3 {
+  SELECT * FROM nm WHERE k=24;
+} {1 {no such table: main.nm_shadow_file}}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 14.0 {
+  CREATE TABLE data(k INTEGER PRIMARY KEY, frame, idx, v BLOB);
+  INSERT INTO data VALUES(1, 1, -1, randomblob(200));
+  INSERT INTO data VALUES(2, 2, -1, randomblob(200));
+  INSERT INTO data VALUES(3, 3, -1, randomblob(200));
+  SELECT zonefile_write('test.zonefile', 'data',
+  '{"encryptionType":"xor","encryptionKey":"pass","debugEncryptionKeyText":1}'
+  );
+
+  CREATE VIRTUAL TABLE nm USING zonefile(cachesize=2);
+  INSERT INTO nm_files(filename,ekey) VALUES('test.zonefile','pass');
+} {{}}
+
+set i 0
+foreach id {1 2 3 2 3 1} {
+  do_execsql_test 14.1.$i {
+    SELECT data.v=nm.v FROM data,nm WHERE data.k=$id AND nm.k=$id
+  } 1
+  incr i
+}
+
+if {[file exists /dev/null]} {
+  do_catchsql_test 14.2 {
+    INSERT INTO nm_files(filename) VALUES('/dev/null');
+  } {1 {failed to read zonefile header from file "/dev/null"}}
+}

 finish_test

@@ -0,0 +1,285 @@
+# 2018 Feb 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the zonefile extension.
+#
+
+package require Tcl 8.6
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source [file join $testdir tester.tcl]
+set testprefix zonefileenc
+
+
+foreach {tn code} {
+  1 { 
+    set K {
+      braking bramble brambles brambly
+      bran branch branched branches
+      branching branchings brand branded
+    }
+    set textkey 1
+  }
+  2 {
+    set K {
+      5e008542742ce0442e37cbf2512e9492    c91c26e0573ca3464e037568c51126da
+      e90e17489c1aef80ac620c9059271a5a    163338707cbe4c72b18d1058a42c5c78
+      5c6b1e7c7c9e8e4a8d8fdc30dfc11bea    ff1012687828ecaac6c9ca86ea0f895e
+      a203f25eb11d4c6afa841dfcf7cd0be0    b6c71e38ca914c460926ef90db39dba0
+      b38255d031d026c258a0a41a9a75d46a    adccca5e5ffa3a7625144a345713aef0
+      cd423b38b73e42ce5894405e6d0e08c0    b460ad2e370a0386726d6ea46e7b0bac
+      503b81de72cb3ef87d9346a850040000    369c290a464a6b88bfd9d1c4755afd42
+      a8a9343efca528f2bf23a972be49dd66    e366b5226bfe3fd0010fa814aae3b996
+      4cad7e80124c2cd447131bae377e60f6    4a0fd2f054e1b08cad0de2dc6aa93246
+      8a23c85e3337da2c97d498f806870fa8    8d14e1f055fd9bec7d07cf0e8baae042
+      7f6954b0dc373028ab3b030aaf44dd58    d220164c3898435a946de6bcbb478cc4
+      566af7ea88ba4ff87fd868e858cf98ea    a5405832235e8f601516f9c49767bdac
+      1bd5b4dc6b54e5ca92ba67d20bf65740    59da30e203bf73840e38e108b83ddb82
+      e516924c2cdf3114f10f2f0e1bdabbc6    b55dd27222a39764222838007e749984
+      190ae9f81b86a5a024e3b97ee2a7121c    469660843a9a9e507d0fb43e92029296
+      e6e600bc063aad12f6387beef650c48a    3097be5c3a52a2f00747587add01b550
+    }
+    set textkey 0
+  }
+} {
+  reset_db
+  load_static_extension db zonefile
+  set nFile 100
+  eval $code
+
+  do_execsql_test 1.$tn.0 {
+    CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+    CREATE TABLE rr(k INTEGER PRIMARY KEY, v);
+  }
+  do_test 1.$tn.1 {
+    for {set i 0} {$i < $nFile} {incr i} {
+      set k [lindex $K [expr $i % [llength $K]]]
+      execsql {
+        DELETE FROM zz;
+        INSERT INTO zz VALUES($i*10+1, 1, -1, randomblob(100));
+        INSERT INTO zz VALUES($i*10+2, 2, -1, randomblob(100));
+        INSERT INTO zz VALUES($i*10+3, 1, -1, randomblob(100));
+        INSERT INTO rr SELECT k,v FROM zz;
+  
+        WITH p(n,v) AS (
+            VALUES('encryptionType', 'xor') UNION ALL
+            VALUES('debugEncryptionKeyText', $textkey) UNION ALL
+            VALUES('encryptionKey', $k)
+        )
+        SELECT zonefile_write('test' || $i || '.zonefile', 'zz', 
+            json_group_object(n, v)
+        ) FROM p;
+      }
+    }
+  } {}
+  
+  proc k {i} { 
+    set val [lindex $::K [expr $i % [llength $::K]]]
+    if {$::textkey==0} {
+      return [binary decode hex $val]
+    }
+    return $val
+  }
+  db func k k
+  
+  do_execsql_test 1.$tn.2 {
+    CREATE VIRTUAL TABLE gg USING zonefile;
+  }
+  for {set i 0} {$i < $nFile} {incr i} {
+    do_execsql_test 1.$tn.2.$i { 
+      INSERT INTO gg_files(filename, ekey) 
+        VALUES('test' || $i || '.zonefile', k($i));
+      SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
+    } 0
+  }
+  
+  db close
+  sqlite3 db test.db
+  load_static_extension db zonefile
+  db func k k
+  
+  do_catchsql_test 1.$tn.3 {
+    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
+  } {1 {missing encryption key for file "test0.zonefile"}}
+  do_execsql_test 1.$tn.4 {
+    UPDATE gg_files SET ekey = k(0) WHERE filename='test0.zonefile';
+  }
+  do_execsql_test 1.$tn.4.2 {
+    SELECT count(*) FROM rr JOIN gg USING(k) 
+      WHERE rr.v==gg.v AND k IN (1,2,3);
+  } {3}
+  do_catchsql_test 1.5 {
+    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
+  } {1 {missing encryption key for file "test1.zonefile"}}
+  
+  do_execsql_test 1.$tn.6 {
+    UPDATE gg_files SET ekey = k(rowid-1);
+  }
+  do_execsql_test 1.$tn.7 {
+    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
+  } {0}
+  do_execsql_test 1.$tn.8 {
+    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v;
+  } {300}
+  
+  forcedelete test.db2
+  do_execsql_test 1.$tn.9.1 {
+    ATTACH 'test.db2' AS maing;
+    CREATE VIRTUAL TABLE maing.g USING zonefile;
+    INSERT INTO g_files(filename) SELECT filename FROM gg_files;
+  }
+  do_catchsql_test 1.$tn.9.2 {
+    SELECT count(*) FROM rr JOIN g USING(k) WHERE rr.v!=g.v;
+  } {1 {missing encryption key for file "test0.zonefile"}}
+  do_execsql_test 1.$tn.9.3 {
+    UPDATE g_files SET ekey = k(rowid-1);
+    SELECT count(*) FROM rr JOIN g USING(k) WHERE rr.v==g.v;
+  } {300}
+  
+  do_execsql_test 1.$tn.10 {
+    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v;
+  } {300}
+}
+#-------------------------------------------------------------------------
+
+reset_db
+load_static_extension db zonefile
+
+do_execsql_test 2.0 {
+  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+}
+foreach {tn alg id} {
+  1 aes_128_ctr 1
+  2 aes_128_cbc 2
+  3 AES_256_CTR 3
+  4 Aes_256_CBC 4
+} {
+  do_catchsql_test 2.1.$tn {
+    WITH p(n,v) AS (
+        VALUES('encryptionType', $alg) UNION ALL
+        VALUES('debugEncryptionKeyText', 1) UNION ALL
+        VALUES('encryptionKey', 'secret')
+    )
+    SELECT zonefile_write('test' || $i || '.zonefile', 'zz', 
+        json_group_object(n, v)
+    ) FROM p;
+  } "1 {unsupported encryption method: $id}"
+}
+
+foreach {tn alg} {
+  1 nosuchmethod! 
+} {
+  do_catchsql_test 2.2.$tn {
+    WITH p(n,v) AS (
+        VALUES('encryptionType', $alg) UNION ALL
+        VALUES('debugEncryptionKeyText', 1) UNION ALL
+        VALUES('encryptionKey', 'secret')
+    )
+    SELECT zonefile_write('test' || $i || '.zonefile', 'zz', 
+        json_group_object(n, v)
+    ) FROM p;
+  } "1 {unknown encryption method: $alg}"
+}
+
+#-------------------------------------------------------------------------
+# Test some hash collisions in the encryption key table.
+#
+
+# This is the same hash function used internally to store keys.
+#
+proc hash {zDb zTab iFile} {
+  binary scan $zDb c*  A
+    binary scan $zTab c* B
+    set h 0
+    foreach i $A { set h [expr ($h + ($h << 3) + $i) & 0xFFFFFFFF] }
+  foreach i $B { set h [expr ($h + ($h << 3) + $i) & 0xFFFFFFFF] }
+  return [expr $h ^ $iFile]
+}
+
+do_test 3.0 {
+  set h1 [expr [hash main zone 1] % 512]
+    for {set i 0} {1} {incr i} {
+      set h2 [expr [hash "aux$i" zone 1] % 512]
+        if {$h1==$h2} break
+    }
+  set i
+} 52
+
+reset_db
+load_static_extension db zonefile
+forcedelete test.db2
+
+do_execsql_test 3.1 {
+  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO zz VALUES(222, -1, -1, randomblob(60));
+  WITH p(n,v) AS (
+      VALUES('encryptionType', 'xor') UNION ALL
+      VALUES('debugEncryptionKeyText', 1) UNION ALL
+      VALUES('encryptionKey', 'pass')
+  )
+  SELECT zonefile_write('test1.zonefile', 'zz', 
+      json_group_object(n, v)
+  ) FROM p;
+
+  DELETE FROM zz;
+  INSERT INTO zz VALUES(333, -1, -1, randomblob(80));
+  WITH p(n,v) AS (
+      VALUES('encryptionType', 'xor') UNION ALL
+      VALUES('debugEncryptionKeyText', 1) UNION ALL
+      VALUES('encryptionKey', 'pass')
+  )
+  SELECT zonefile_write('test2.zonefile', 'zz', 
+      json_group_object(n, v)
+  ) FROM p;
+} {{} {}}
+
+do_execsql_test 3.2 {
+  ATTACH 'test.db2' AS aux52;
+  CREATE VIRTUAL TABLE main.zone USING zonefile;
+  CREATE VIRTUAL TABLE aux52.zone USING zonefile;
+  INSERT INTO main.zone_files(filename, ekey) VALUES('test1.zonefile', 'pass');
+  INSERT INTO aux52.zone_files(filename, ekey) VALUES('test2.zonefile', 'pass');
+}
+
+do_execsql_test 3.3 {
+  SELECT v IS NULL FROM main.zone;
+  SELECT v IS NULL FROM aux52.zone;
+} {0 0}
+
+do_test 3.4 {
+  set h1 [expr [hash main zone 1] % 512]
+  for {set i 0} {1} {incr i} {
+    set h2 [expr [hash "aux$i" zone 2] % 512]
+    if {$h1==$h2} break
+  }
+  set i
+} 682
+
+forcedelete test.db3
+do_execsql_test 3.5 {
+  ATTACH 'test.db3' AS aux682;
+  CREATE VIRTUAL TABLE aux682.zone USING zonefile;
+  INSERT INTO aux682.zone_files(filename, ekey) VALUES('test1.zonefile','pass');
+  INSERT INTO aux682.zone_files(filename, ekey) VALUES('test2.zonefile','pass');
+  INSERT INTO main.zone_files(filename, ekey) VALUES('test2.zonefile','pass');
+}
+
+do_execsql_test 3.6 {
+  SELECT v IS NULL FROM main.zone;
+  SELECT v IS NULL FROM aux682.zone;
+  SELECT v IS NULL FROM main.zone;
+} {0 0 0 0 0 0}
+
+
+finish_test
+
@@ -0,0 +1,251 @@
+# 2018 Feb 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the zonefile extension.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source [file join $testdir tester.tcl]
+source [file join $testdir malloc_common.tcl]
+set testprefix zonefilefault
+load_static_extension db zonefile
+
+do_execsql_test 1.0 {
+  CREATE TABLE tt(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO tt VALUES(1, -1, -1, randomblob(100));
+  INSERT INTO tt VALUES(2, -1, -1, randomblob(200));
+  INSERT INTO tt VALUES(3,  1, -1, randomblob(300));
+  INSERT INTO tt VALUES(4,  2, -1, randomblob(400));
+}
+
+do_faultsim_test 1.1 -faults oom* -prep {
+  sqlite3 db test.db
+  load_static_extension db zonefile
+} -body {
+  execsql {
+    SELECT zonefile_write('test.zonefile', 'tt',
+        '{"encryptionType":"xor", "encryptionKey":"secret",
+          "debugEncryptionKeyText":1
+        }'
+    );
+  }
+} -test {
+  faultsim_test_result {0 {{}}}
+}
+
+set sql {
+    SELECT zonefile_write('test.zonefile', 'tt',
+        '{"compressionTypeContent":"zstd_global_dict"}'
+    );
+}
+set HAVE_ZSTD 0
+if {[catch { db eval $sql }]==0} {
+  set HAVE_ZSTD 1
+}
+
+if {$HAVE_ZSTD} {
+  do_faultsim_test 1.2 -faults oom* -prep {
+    sqlite3 db test.db
+    load_static_extension db zonefile
+  } -body {
+    execsql {
+      SELECT zonefile_write('test.zonefile', 'tt',
+          '{"compressionTypeContent":"zstd_global_dict"}'
+      );
+    }
+  } -test {
+    faultsim_test_result {0 {{}}}
+  }
+}
+
+do_execsql_test 1.3.0 { UPDATE tt SET frame = NULL; }
+do_faultsim_test 1.3 -faults oom* -prep {
+  sqlite3 db test.db
+  load_static_extension db zonefile
+} -body {
+  execsql {
+    SELECT zonefile_write('test.zonefile', 'tt');
+  }
+} -test {
+  faultsim_test_result {0 {{}}}
+}
+
+do_execsql_test 1.4.0 {
+  INSERT INTO tt VALUES(5, -1, -1, randomblob(100));
+  INSERT INTO tt VALUES(6, -1, -1, randomblob(100));
+  INSERT INTO tt VALUES(7, -1, -1, randomblob(100));
+  INSERT INTO tt VALUES(8, -1, -1, randomblob(100));
+  INSERT INTO tt VALUES(9, -1, -1, randomblob(100));
+  CREATE VIRTUAL TABLE ttz USING zonefile;
+}
+if {$HAVE_ZSTD} {
+  faultsim_save_and_close
+  do_faultsim_test 1.4 -faults oom* -prep {
+    faultsim_restore_and_reopen
+    load_static_extension db zonefile
+  } -body {
+    execsql {
+      SELECT zonefile_write('test.zonefile', 'tt',
+          '{"compressionTypeIndexData":"zstd"}'
+      );
+    }
+  } -test {
+    faultsim_test_result {0 {{}}}
+  }
+
+  faultsim_save_and_close
+  do_faultsim_test 1.5 -faults oom* -prep {
+    faultsim_restore_and_reopen
+    load_static_extension db zonefile
+  } -body {
+    execsql {
+      INSERT INTO ttz_files(filename) VALUES('test.zonefile');
+    }
+  } -test {
+    faultsim_test_result {0 {}}
+  }
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+  SELECT zonefile_write('test.zonefile', 'tt',
+      '{"encryptionType":"xor", "encryptionKey":"secret",
+          "debugEncryptionKeyText":1
+      }'
+  );
+  CREATE VIRTUAL TABLE zz USING zonefile;
+} {{}}
+
+faultsim_save_and_close
+do_faultsim_test 2.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql {
+    INSERT INTO zz_files(filename, ekey) VALUES('test.zonefile', 'secret')
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+faultsim_save_and_close
+do_faultsim_test 2.2 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql { SELECT json_extract(header, '$.magicNumber') FROM zz_files }
+} -test {
+  faultsim_test_result {0 1179332920}
+}
+
+do_faultsim_test 2.3 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql { DELETE FROM zz_files }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+faultsim_save_and_close
+do_faultsim_test 3.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql { CREATE VIRTUAL TABLE t1 USING zonefile(cachesize=5) }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+faultsim_save_and_close
+do_faultsim_test 3.2 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql { DROP TABLE t1 }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 4.0 {
+  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO zz VALUES(1, -1, -1, randomblob(100));
+  INSERT INTO zz VALUES(2, -1, -1, randomblob(100));
+  INSERT INTO zz VALUES(3, -1, -1, randomblob(100));
+  SELECT zonefile_write('test.zonefile', 'zz');
+  CREATE VIRTUAL TABLE zone USING zonefile;
+  INSERT INTO zone_files(filename) VALUES('test.zonefile');
+} {{}}
+
+faultsim_save_and_close
+do_faultsim_test 4.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql { SELECT v IS NULL FROM zone WHERE k = 2 }
+} -test {
+  faultsim_test_result {0 0}
+}
+
+
+if {$HAVE_ZSTD} {
+  set params {
+      {"encryptionType":"xor","encryptionKey":"pass",
+       "compressionTypeContent":"zstd_global_dict",
+          "debugEncryptionKeyText":1
+      }
+  }
+} else {
+  set params { 
+      {"encryptionType":"xor","encryptionKey":"pass",
+          "debugEncryptionKeyText":1
+      }
+  }
+}
+do_execsql_test 4.2 {
+  SELECT zonefile_write('test.zonefile', 'zz', $params);
+  CREATE VIRTUAL TABLE zone2 USING zonefile;
+  INSERT INTO zone2_files(filename,ekey) VALUES('test.zonefile','pass');
+} {{}}
+
+faultsim_save_and_close
+do_faultsim_test 4.3 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+  execsql { UPDATE zone2_files SET ekey='pass' }
+} -body {
+  execsql { SELECT v IS NULL FROM zone2 WHERE k = 2 }
+} -test {
+  faultsim_test_result {0 0}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_faultsim_test 4.4 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  load_static_extension db zonefile
+} -test {
+  faultsim_test_result {0 {}} {1 {initialization of zonefile failed: }}
+}
+
+finish_test
+
@@ -520,6 +520,7 @@ SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 DBFUZZ_OPT =
 KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
 ST_OPT = -DSQLITE_THREADSAFE=0
@@ -1,11 +1,11 @@
-C Fix\sanother\spoint\sin\szonefile.c\sso\sthat\sall\sfiles\sare\sopened\sin\seither\s"rb"\sor\n"wb"\smode.
-D 2018-02-15T15:24:12.092
+C Update\szonefile\sREADME.md\sfile\sto\smention\sthe\sframe\scache.
+D 2018-02-27T20:09:45.338
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
+F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc b5e9eb7212c5a61d4800172c710376d1158fd5eea2edc0be8e05c0c372e767e4
-F README.md d748f58e3ab0fe0307fb4ae0942b415d93dcc4288756e366cc9e7cf8260c093f
+F Makefile.msc 738b130a37c8855f46c519cdeab968a483042017b8e90a04a8156e26d6a3769d
+F README.md 1d5342ebda97420f114283e604e5fe99b0da939d63b76d492eabbaae23488276
 F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
@@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d
 F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c
-F ext/fts3/fts3_write.c 7a7cf93c02ebe0ee4211e4aa07da77586c2dcf7d381c1382f81d29c9aa8cae8c
+F ext/fts3/fts3_write.c b583dede85eb0c3c3026f8d7ccb781ea4e845ae583754fecb2ca425b5907d87d
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
@@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
-F ext/misc/spellfix.c 705ce1ed701b38f43c8ee7dd848877ff51c09eda322e39ccac6873adb4b74735
+F ext/misc/spellfix.c b3a644285cb008f3c10ed4cf04e17adcbc7d62c3911c79d786dfc91c177534f0
 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 3c3e21d0351a44c34c67ca7e833ba04e0de34421b4c9359aa764919ee36b37ef
+F ext/misc/zipfile.c bc88dbf9aa26c24595d158c1398208786ef34e7aee19ea29e8f0110c752fca03
 F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
@@ -408,13 +408,15 @@ F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
-F ext/zonefile/README.md 387ad2b748e98eeea21fd4dbb609fefe313263fadb3fc6c01c512b4c95e55ae4
-F ext/zonefile/zonefile.c 88d4ba281f47d35ce5989a7c500cfd05e3c8e348426baebcb7bcef314aee08f2
-F ext/zonefile/zonefile1.test 872ec8d549af0f1423601acdfe29390803fab3e14b8d5715f2f4bbd11431f1f5
+F ext/zonefile/README.md 49bc392fe0dde788cc63b2ea71e26cfba02141b6ac192e245668d0b7b43b5750
+F ext/zonefile/zonefile.c ce917d60a62c8e384087954f482ab703b75a0660385e7a685bcafc0cff1c3038
+F ext/zonefile/zonefile1.test 3719e1d069064f4e12e915f8d9f6256c769b1582258b3997e0492076a01d5f36
+F ext/zonefile/zonefileenc.test e8624853ac224bfdeab8d2a231796499c3032e2a8bb1d2208f52fddf9507e38f
+F ext/zonefile/zonefilefault.test 6df281fc1ec7859d4ae2a3133d8783184a263edf6d081218958b6ae86b22ee13
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 0efdf6ac125eb810494d72dd7016ab07ddf3e3fa8615a14aef8491b69818fbe4
+F main.mk e5ac915f9959445602c07fd5aee00c5c09b4616658e67aa12e765399cd686763
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -428,12 +430,12 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594
 F src/analyze.c 6b42e36a5dcc2703a771f2411bd5e99524bd62c7ecde209bb88dfb04c72f046e
-F src/attach.c 84c477e856b24c2b9a0983b438a707c0cf4d616cee7a425401d418e58afec24c
+F src/attach.c 79cb6b365d79b96b15dd429aa3e4c3b99335039e059baca8df28f1708fbff261
 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c f7bb37c27b3734d58796febb30b5d818b3b4eb592b57b88039e5d92b3ebd9dab
+F src/btree.c 18a53540aa35dbdf77f715ea928422a4ed9011dc16ea7b50f803fd1617fcc4f5
 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
 F src/build.c 672022c06e1a5c2653f80c77a687de11f7e65ce81d20fe2825aadfa13a875c33
@@ -448,7 +450,7 @@ F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
-F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
+F src/global.c 01506976bd75e5e7b977207a6a05062e2dd0050012f8071be06bbea22ec6d69a
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
@@ -468,14 +470,14 @@ F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81
 F src/mutex.c b021263554c8a3995e9d53193b8194b96d1ed28e06c3b532dd7f7d29cf0c7d53
 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
-F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23
-F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d
+F src/mutex_unix.c aaf9ebc3f89df28483c52208497a99a02cc3650011422fc9d4c57e4392f7fe58
+F src/mutex_w32.c 7670d770c94bbfe8289bec9d7f1394c5a00a57c37f892aab6b6612d085255235
 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7
 F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f
+F src/os_unix.c 928586cbde31890538854f1657463392d63b2d7794f577774573ae53ec283d7b
 F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89
@@ -487,16 +489,16 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
-F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
+F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
-F src/shell.c.in 0b2c8d55e3703723a22610d75ccd4037962c3bcb5ad5570efd781e8c521811c4
-F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e
+F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1
+F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
-F src/sqliteInt.h 119e240796d23054148005524f1d54a61e27dbef53bef3bc7ab65001405f4751
+F src/sqliteInt.h fd8943e9d6b8bd04b3a22ef806b843c2c06f022db3f37f1436ac3a9b04e5b8a1
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -516,7 +518,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
-F src/test_config.c cc8a1d44648d9392a14f4ecfc841d027daaf61f952b9f70792edf11373aaa3dd
+F src/test_config.c 03df8eb9dc8d5ee96588dfa34fd989c75625d5a0daf59491e32b966da4010870
 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
@@ -559,8 +561,8 @@ F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
-F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c a6892805df427bb9d554c37641406438bff568f99d26e72c1fb7e45f2ed6ce5b
+F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5
+F src/vdbe.c 72f5a4b95be486170104779aeef000423acabf55ac0c661afdae889be72c8922
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
@@ -571,13 +573,13 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
+F src/wal.c 0ba9af2589b67c4bc60c773c7d2080491099037af21e73545f932bb29bf70b65
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
-F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3
-F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb
+F src/wherecode.c e1aaadd8fec650037cfbf27d1b3470338fb3b58fec34d11082df16fe9a08fbd7
+F src/whereexpr.c 53532be687e12f3cd314f1e204cd4fbdac7ad250e918a182b048121e16e828ae
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -921,7 +923,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0
 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
 F test/fts4noti.test 5553d7bb2e20bf4a06b23e849352efc022ce6309
-F test/fts4onepass.test 7319d61a2ed1325fc54afd0c060a0513b462303a
+F test/fts4onepass.test d69ddc4ee3415e40b0c5d1d0408488a87614d4f63ba9c44f3e52db541d6b7cc7
 F test/fts4opt.test fd6a11684b965e1999564ae763797b7fb9e34c96
 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
@@ -930,7 +932,7 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
 F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4
-F test/func6.test 612311a51adad23326d15353eea8d90394798acb26dd1f7c7cad59bd8ac982d2
+F test/func6.test 90e42b64c4f9fb6f04f44cb8a1da586c8542502e926b19c76504fe74ff2a9b7c
 F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa
 F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1
 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
@@ -1144,7 +1146,7 @@ F test/pragma4.test 3046501bee2f652dc2a4f9c87781e2741361d6864439c8381aba6c3b774b
 F test/pragma5.test 824ce6ced5d6b7ec71abe37fc6005ff836fe39d638273dc5192b39864b9ee983
 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8
 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc
-F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad
+F test/printf2.test 30b5dd0b4b992dc5626496846ecce17ff592cacbcb11c3e589f3ac4d7e129dae
 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb
 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
 F test/pushdown.test 5e72c51c5e33253ed639ccee1e01ce62d62b6eee5ca893cd82334e4ee7b1d7fc
@@ -1169,7 +1171,7 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a
 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc
 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81
 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d
-F test/rowvalue.test e1bd1690d891abff6138f9612241615d2de7671cb28eb58ebdd591a5b01089a4
+F test/rowvalue.test 32861d6a933ded868035f2ec79aeb993a2a46eb7a6d282ae13415a4c2e369463
 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b
 F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256
 F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea
@@ -1265,9 +1267,10 @@ F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
 F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c39603375b8
-F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db
+F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
+F test/spellfix4.test 51c7c26514ade169855c66bcf130bd5acfb4d7fd090cc624645ab275ae6a41fb
 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e
 F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142
 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a
@@ -1596,6 +1599,7 @@ F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc
 F test/with1.test ca08e291249a810a2ec9b72ceef5575e07d5925b360fcf6652ae6fe06ac4dced
 F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab
 F test/with3.test e71604a0e53cba82bc04c703987cb1d6751ec0b6
+F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205
 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64
 F test/without_rowid1.test 06b7215130882d6a072233820dd364c874c4fd69221e8fc756ec471009192874
 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99
@@ -1666,7 +1670,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c
 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec
-F tool/speed-check.sh 9ae425da8819e54e780cf494fc6d8175dfb16e109ae3214a45a5c9bb2b74e2c4
+F tool/speed-check.sh 4ff9b095cf1a7643f0264e7fb7d23f0b12b7cce587a9de315877c378e90eeaf4
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
@@ -1708,7 +1712,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4bb854ddd9c1dc2972fd4f7c2c2b2d121caa662d5085694c2dbb35d331a61444
-R 2c56f94b965e90e1ed8c01e5183c20c8
+P f11beb16a87cc967e896cf67121b1e4045e427ebdc6a424e9f009ffced955d36
+R ce086be2fa4585d5057d94c9c14e09e6
 U dan
-Z 2cff9a10fd106f92fe182af0aacf7c32
+Z fcd96707fe4566207f6172155f7ee098
@@ -1 +1 @@
-fb1c2277912c55cfae30c18b5434bc193748746395fa7df983cd8a29e5741ff9
+84e9351bbd0a9bf9e1d64a441366a90276c231b44548cbc947f7fb2842f82490
@@ -504,6 +504,14 @@ int sqlite3FixSelect(
    if( sqlite3FixExpr(pFix, pSelect->pLimit) ){
      return 1;
    }
+    if( pSelect->pWith ){
+      int i;
+      for(i=0; i<pSelect->pWith->nCte; i++){
+        if( sqlite3FixSelect(pFix, pSelect->pWith->a[i].pSelect) ){
+          return 1;
+        }
+      }
+    }
    pSelect = pSelect->pPrior;
  }
  return 0;
@@ -4388,11 +4388,19 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){
 ** Using this cache reduces the number of calls to btreeParseCell().
 */
 #ifndef NDEBUG
+  static int cellInfoEqual(CellInfo *a, CellInfo *b){
+    if( a->nKey!=b->nKey ) return 0;
+    if( a->pPayload!=b->pPayload ) return 0;
+    if( a->nPayload!=b->nPayload ) return 0;
+    if( a->nLocal!=b->nLocal ) return 0;
+    if( a->nSize!=b->nSize ) return 0;
+    return 1;
+  }
  static void assertCellInfo(BtCursor *pCur){
    CellInfo info;
    memset(&info, 0, sizeof(info));
    btreeParseCell(pCur->pPage, pCur->ix, &info);
-    assert( CORRUPT_DB || memcmp(&info, &pCur->info, sizeof(info))==0 );
+    assert( CORRUPT_DB || cellInfoEqual(&info, &pCur->info) );
  }
 #else
  #define assertCellInfo(x)
@@ -258,6 +258,13 @@ const Token sqlite3IntTokens[] = {
   { "1", 1 }
 };

+#ifdef VDBE_PROFILE
+/*
+** The following performance counter can be used in place of
+** sqlite3Hwtime() for profiling.  This is a no-op on standard builds.
+*/
+sqlite3_uint64 sqlite3NProfileCnt = 0;
+#endif

 /*
 ** The value of the "pending" byte must be 0x40000000 (1 byte past the
@@ -50,11 +50,12 @@ struct sqlite3_mutex {
 #endif
 };
 #if SQLITE_MUTEX_NREF
-#define SQLITE3_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER,0,0,(pthread_t)0,0}
+# define SQLITE3_MUTEX_INITIALIZER(id) \
+     {PTHREAD_MUTEX_INITIALIZER,id,0,(pthread_t)0,0}
 #elif defined(SQLITE_ENABLE_API_ARMOR)
-#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0 }
+# define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER, id }
 #else
-#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }
+#define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER }
 #endif

 /*
@@ -151,18 +152,18 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; }
 */
 static sqlite3_mutex *pthreadMutexAlloc(int iType){
  static sqlite3_mutex staticMutexes[] = {
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER
+    SQLITE3_MUTEX_INITIALIZER(2),
+    SQLITE3_MUTEX_INITIALIZER(3),
+    SQLITE3_MUTEX_INITIALIZER(4),
+    SQLITE3_MUTEX_INITIALIZER(5),
+    SQLITE3_MUTEX_INITIALIZER(6),
+    SQLITE3_MUTEX_INITIALIZER(7),
+    SQLITE3_MUTEX_INITIALIZER(8),
+    SQLITE3_MUTEX_INITIALIZER(9),
+    SQLITE3_MUTEX_INITIALIZER(10),
+    SQLITE3_MUTEX_INITIALIZER(11),
+    SQLITE3_MUTEX_INITIALIZER(12),
+    SQLITE3_MUTEX_INITIALIZER(13)
  };
  sqlite3_mutex *p;
  switch( iType ){
@@ -180,6 +181,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){
        pthread_mutexattr_settype(&recursiveAttr, PTHREAD_MUTEX_RECURSIVE);
        pthread_mutex_init(&p->mutex, &recursiveAttr);
        pthread_mutexattr_destroy(&recursiveAttr);
+#endif
+#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR)
+        p->id = SQLITE_MUTEX_RECURSIVE;
 #endif
      }
      break;
@@ -188,6 +192,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){
      p = sqlite3MallocZero( sizeof(*p) );
      if( p ){
        pthread_mutex_init(&p->mutex, 0);
+#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR)
+        p->id = SQLITE_MUTEX_FAST;
+#endif
      }
      break;
    }
@@ -203,7 +210,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){
    }
  }
 #if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR)
-  if( p ) p->id = iType;
+  assert( p==0 || p->id==iType );
 #endif
  return p;
 }
@@ -40,7 +40,7 @@ struct sqlite3_mutex {
 #ifdef SQLITE_DEBUG
  volatile int nRef;         /* Number of enterances */
  volatile DWORD owner;      /* Thread holding this mutex */
-  volatile int trace;        /* True to trace changes */
+  volatile LONG trace;       /* True to trace changes */
 #endif
 };

@@ -52,10 +52,10 @@ struct sqlite3_mutex {
 #define SQLITE_W32_MUTEX_INITIALIZER { 0 }

 #ifdef SQLITE_DEBUG
-#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, \
+#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id, \
                                    0L, (DWORD)0, 0 }
 #else
-#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 }
+#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id }
 #endif

 #ifdef SQLITE_DEBUG
@@ -98,18 +98,18 @@ void sqlite3MemoryBarrier(void){
 ** Initialize and deinitialize the mutex subsystem.
 */
 static sqlite3_mutex winMutex_staticMutexes[] = {
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER
+  SQLITE3_MUTEX_INITIALIZER(2),
+  SQLITE3_MUTEX_INITIALIZER(3),
+  SQLITE3_MUTEX_INITIALIZER(4),
+  SQLITE3_MUTEX_INITIALIZER(5),
+  SQLITE3_MUTEX_INITIALIZER(6),
+  SQLITE3_MUTEX_INITIALIZER(7),
+  SQLITE3_MUTEX_INITIALIZER(8),
+  SQLITE3_MUTEX_INITIALIZER(9),
+  SQLITE3_MUTEX_INITIALIZER(10),
+  SQLITE3_MUTEX_INITIALIZER(11),
+  SQLITE3_MUTEX_INITIALIZER(12),
+  SQLITE3_MUTEX_INITIALIZER(13)
 };

 static int winMutex_isInit = 0;
@@ -239,15 +239,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){
      }
 #endif
      p = &winMutex_staticMutexes[iType-2];
-      p->id = iType;
 #ifdef SQLITE_DEBUG
 #ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC
-      p->trace = 1;
+      InterlockedCompareExchange(&p->trace, 1, 0);
 #endif
 #endif
      break;
    }
  }
+  assert( p==0 || p->id==iType );
  return p;
 }

@@ -468,7 +468,11 @@ static struct unix_syscall {
 #endif
 #define osFchown    ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent)

+#if defined(HAVE_FCHOWN)
  { "geteuid",      (sqlite3_syscall_ptr)geteuid,         0 },
+#else
+  { "geteuid",      (sqlite3_syscall_ptr)0,               0 },
+#endif
 #define osGeteuid   ((uid_t(*)(void))aSyscall[21].pCurrent)

 #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0
@@ -4176,12 +4180,10 @@ static int unixShmSystemLock(

  if( pShmNode->h>=0 ){
    /* Initialize the locking parameters */
-    memset(&f, 0, sizeof(f));
    f.l_type = lockType;
    f.l_whence = SEEK_SET;
    f.l_start = ofst;
    f.l_len = n;
-
    rc = osFcntl(pShmNode->h, F_SETLK, &f);
    rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY;
  }
@@ -206,6 +206,11 @@ void sqlite3VXPrintf(
  PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */
  char buf[etBUFSIZE];       /* Conversion buffer */

+  /* pAccum never starts out with an empty buffer that was obtained from 
+  ** malloc().  This precondition is required by the mprintf("%z...")
+  ** optimization. */
+  assert( pAccum->nChar>0 || (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 );
+
  bufpt = 0;
  if( (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC)!=0 ){
    pArgList = va_arg(ap, PrintfArguments*);
@@ -624,9 +629,38 @@ void sqlite3VXPrintf(
      case etCHARX:
        if( bArgList ){
          bufpt = getTextArg(pArgList);
-          c = bufpt ? bufpt[0] : 0;
+          length = 1;
+          if( bufpt ){
+            buf[0] = c = *(bufpt++);
+            if( (c&0xc0)==0xc0 ){
+              while( length<4 && (bufpt[0]&0xc0)==0x80 ){
+                buf[length++] = *(bufpt++);
+              }
+            }
+          }else{
+            buf[0] = 0;
+          }
        }else{
-          c = va_arg(ap,int);
+          unsigned int ch = va_arg(ap,unsigned int);
+          if( ch<0x00080 ){
+            buf[0] = ch & 0xff;
+            length = 1;
+          }else if( ch<0x00800 ){
+            buf[0] = 0xc0 + (u8)((ch>>6)&0x1f);
+            buf[1] = 0x80 + (u8)(ch & 0x3f);
+            length = 2;
+          }else if( ch<0x10000 ){
+            buf[0] = 0xe0 + (u8)((ch>>12)&0x0f);
+            buf[1] = 0x80 + (u8)((ch>>6) & 0x3f);
+            buf[2] = 0x80 + (u8)(ch & 0x3f);
+            length = 3;
+          }else{
+            buf[0] = 0xf0 + (u8)((ch>>18) & 0x07);
+            buf[1] = 0x80 + (u8)((ch>>12) & 0x3f);
+            buf[2] = 0x80 + (u8)((ch>>6) & 0x3f);
+            buf[3] = 0x80 + (u8)(ch & 0x3f);
+            length = 4;
+          }
        }
        if( precision>1 ){
          width -= precision-1;
@@ -634,12 +668,13 @@ void sqlite3VXPrintf(
            sqlite3AppendChar(pAccum, width-1, ' ');
            width = 0;
          }
-          sqlite3AppendChar(pAccum, precision-1, c);
+          while( precision-- > 1 ){
+            sqlite3StrAccumAppend(pAccum, buf, length);
+          }
        }
-        length = 1;
-        buf[0] = c;
        bufpt = buf;
-        break;
+        flag_altform2 = 1;
+        goto adjust_width_for_utf8;
      case etSTRING:
      case etDYNSTRING:
        if( bArgList ){
@@ -651,17 +686,45 @@ void sqlite3VXPrintf(
        if( bufpt==0 ){
          bufpt = "";
        }else if( xtype==etDYNSTRING ){
+          if( pAccum->nChar==0 && pAccum->mxAlloc && width==0 && precision<0 ){
+            /* Special optimization for sqlite3_mprintf("%z..."):
+            ** Extend an existing memory allocation rather than creating
+            ** a new one. */
+            assert( (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 );
+            pAccum->zText = bufpt;
+            pAccum->nAlloc = sqlite3DbMallocSize(pAccum->db, bufpt);
+            pAccum->nChar = 0x7fffffff & (int)strlen(bufpt);
+            pAccum->printfFlags |= SQLITE_PRINTF_MALLOCED;
+            length = 0;
+            break;
+          }
          zExtra = bufpt;
        }
        if( precision>=0 ){
-          for(length=0; length<precision && bufpt[length]; length++){}
+          if( flag_altform2 ){
+            /* Set length to the number of bytes needed in order to display
+            ** precision characters */
+            unsigned char *z = (unsigned char*)bufpt;
+            while( precision-- > 0 && z[0] ){
+              SQLITE_SKIP_UTF8(z);
+            }
+            length = (int)(z - (unsigned char*)bufpt);
+          }else{
+            for(length=0; length<precision && bufpt[length]; length++){}
+          }
        }else{
          length = 0x7fffffff & (int)strlen(bufpt);
        }
+      adjust_width_for_utf8:
+        if( flag_altform2 && width>0 ){
+          /* Adjust width to account for extra bytes in UTF-8 characters */
+          int ii = length - 1;
+          while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++;
+        }
        break;
-      case etSQLESCAPE:           /* Escape ' characters */
-      case etSQLESCAPE2:          /* Escape ' and enclose in '...' */
-      case etSQLESCAPE3: {        /* Escape " characters */
+      case etSQLESCAPE:           /* %q: Escape ' characters */
+      case etSQLESCAPE2:          /* %Q: Escape ' and enclose in '...' */
+      case etSQLESCAPE3: {        /* %w: Escape " characters */
        int i, j, k, n, isnull;
        int needQuote;
        char ch;
@@ -675,9 +738,17 @@ void sqlite3VXPrintf(
        }
        isnull = escarg==0;
        if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)");
+        /* For %q, %Q, and %w, the precision is the number of byte (or
+        ** characters if the ! flags is present) to use from the input.
+        ** Because of the extra quoting characters inserted, the number
+        ** of output characters may be larger than the precision.
+        */
        k = precision;
        for(i=n=0; k!=0 && (ch=escarg[i])!=0; i++, k--){
          if( ch==q )  n++;
+          if( flag_altform2 && (ch&0xc0)==0xc0 ){
+            while( (escarg[i+1]&0xc0)==0x80 ){ i++; }
+          }
        }
        needQuote = !isnull && xtype==etSQLESCAPE2;
        n += i + 3;
@@ -700,10 +771,7 @@ void sqlite3VXPrintf(
        if( needQuote ) bufpt[j++] = q;
        bufpt[j] = 0;
        length = j;
-        /* The precision in %q and %Q means how many input characters to
-        ** consume, not the length of the output...
-        ** if( precision>=0 && precision<length ) length = precision; */
-        break;
+        goto adjust_width_for_utf8;
      }
      case etTOKEN: {
        Token *pToken;
@@ -742,7 +810,10 @@ void sqlite3VXPrintf(
    /*
    ** The text of the conversion is pointed to by "bufpt" and is
    ** "length" characters long.  The field width is "width".  Do
-    ** the output.
+    ** the output.  Both length and width are in bytes, not characters,
+    ** at this point.  If the "!" flag was present on string conversions
+    ** indicating that width and precision should be expressed in characters,
+    ** then the values have been translated prior to reaching this point.
    */
    width -= length;
    if( width>0 ){
@@ -1069,6 +1069,7 @@ struct ShellState {
 #define SHELL_OPEN_NORMAL     1      /* Normal database file */
 #define SHELL_OPEN_APPENDVFS  2      /* Use appendvfs */
 #define SHELL_OPEN_ZIPFILE    3      /* Use the zipfile virtual table */
+#define SHELL_OPEN_READONLY   4      /* Open a normal database read-only */

 /*
 ** These are the allowed shellFlgs values
@@ -3258,6 +3259,7 @@ static char zHelp[] =
  "                         on the output.\n"
  ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE\n"
  "                         The --new option starts with an empty file\n"
+  "                         Other options: --readonly --append --zip\n"
  ".output ?FILE?         Send output to FILE or stdout\n"
  ".print STRING...       Print literal STRING\n"
  ".prompt MAIN CONTINUE  Replace the standard prompts\n"
@@ -3454,6 +3456,10 @@ static void open_db(ShellState *p, int keepAlive){
        sqlite3_open(":memory:", &p->db);
        break;
      }
+      case SHELL_OPEN_READONLY: {
+        sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READONLY, 0);
+        break;
+      }
      case SHELL_OPEN_UNSPEC:
      case SHELL_OPEN_NORMAL: {
        sqlite3_open(p->zDbFilename, &p->db);
@@ -6307,6 +6313,8 @@ static int do_meta_command(char *zLine, ShellState *p){
 #endif
      }else if( optionMatch(z, "append") ){
        p->openMode = SHELL_OPEN_APPENDVFS;
+      }else if( optionMatch(z, "readonly") ){
+        p->openMode = SHELL_OPEN_READONLY;
      }else if( z[0]=='-' ){
        utf8_printf(stderr, "unknown option: %s\n", z);
        rc = 1;
@@ -7985,6 +7993,7 @@ static const char zOptions[] =
  "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
  "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
  "   -quote               set output mode to 'quote'\n"
+  "   -readonly            open the database read-only\n"
  "   -separator SEP       set output column separator. Default: '|'\n"
  "   -stats               print memory stats before each finalize\n"
  "   -version             show SQLite version\n"
@@ -8094,21 +8103,39 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
  }
 #endif
  main_init(&data);
-  sqlite3_initialize();
+
+  /* On Windows, we must translate command-line arguments into UTF-8.
+  ** The SQLite memory allocator subsystem has to be enabled in order to
+  ** do this.  But we want to run an sqlite3_shutdown() afterwards so that
+  ** subsequent sqlite3_config() calls will work.  So copy all results into
+  ** memory that does not come from the SQLite memory allocator.
+  */
 #if !SQLITE_SHELL_IS_UTF8
-  argv = sqlite3_malloc64(sizeof(argv[0])*argc);
+  sqlite3_initialize();
+  argv = malloc(sizeof(argv[0])*argc);
  if( argv==0 ){
    raw_printf(stderr, "out of memory\n");
    exit(1);
  }
  for(i=0; i<argc; i++){
-    argv[i] = sqlite3_win32_unicode_to_utf8(wargv[i]);
+    char *z = sqlite3_win32_unicode_to_utf8(wargv[i]);
+    int n;
+    if( z==0 ){
+      raw_printf(stderr, "out of memory\n");
+      exit(1);
+    }
+    n = (int)strlen(z);
+    argv[i] = malloc( n+1 );
    if( argv[i]==0 ){
      raw_printf(stderr, "out of memory\n");
      exit(1);
    }
+    memcpy(argv[i], z, n+1);
+    sqlite3_free(z);
  }
+  sqlite3_shutdown();
 #endif
+
  assert( argc>=1 && argv && argv[0] );
  Argv0 = argv[0];

@@ -8234,6 +8261,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #endif
    }else if( strcmp(z,"-append")==0 ){
      data.openMode = SHELL_OPEN_APPENDVFS;
+    }else if( strcmp(z,"-readonly")==0 ){
+      data.openMode = SHELL_OPEN_READONLY;
    }
  }
  if( data.zDbFilename==0 ){
@@ -2496,16 +2496,16 @@ void sqlite3_free_table(char **result);
 **
 ** These routines are work-alikes of the "printf()" family of functions
 ** from the standard C library.
-** These routines understand most of the common K&R formatting options,
-** plus some additional non-standard formats, detailed below.
-** Note that some of the more obscure formatting options from recent
-** C-library standards are omitted from this implementation.
+** These routines understand most of the common formatting options from
+** the standard library printf() 
+** plus some additional non-standard formats ([%q], [%Q], [%w], and [%z]).
+** See the [built-in printf()] documentation for details.
 **
 ** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their
-** results into memory obtained from [sqlite3_malloc()].
+** results into memory obtained from [sqlite3_malloc64()].
 ** The strings returned by these two routines should be
 ** released by [sqlite3_free()].  ^Both routines return a
-** NULL pointer if [sqlite3_malloc()] is unable to allocate enough
+** NULL pointer if [sqlite3_malloc64()] is unable to allocate enough
 ** memory to hold the resulting string.
 **
 ** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from
@@ -2529,71 +2529,7 @@ void sqlite3_free_table(char **result);
 **
 ** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf().
 **
-** These routines all implement some additional formatting
-** options that are useful for constructing SQL statements.
-** All of the usual printf() formatting options apply.  In addition, there
-** is are "%q", "%Q", "%w" and "%z" options.
-**
-** ^(The %q option works like %s in that it substitutes a nul-terminated
-** string from the argument list.  But %q also doubles every '\'' character.
-** %q is designed for use inside a string literal.)^  By doubling each '\''
-** character it escapes that character and allows it to be inserted into
-** the string.
-**
-** For example, assume the string variable zText contains text as follows:
-**
-** <blockquote><pre>
-**  char *zText = "It's a happy day!";
-** </pre></blockquote>
-**
-** One can use this text in an SQL statement as follows:
-**
-** <blockquote><pre>
-**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES('%q')", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-** </pre></blockquote>
-**
-** Because the %q format string is used, the '\'' character in zText
-** is escaped and the SQL generated is as follows:
-**
-** <blockquote><pre>
-**  INSERT INTO table1 VALUES('It''s a happy day!')
-** </pre></blockquote>
-**
-** This is correct.  Had we used %s instead of %q, the generated SQL
-** would have looked like this:
-**
-** <blockquote><pre>
-**  INSERT INTO table1 VALUES('It's a happy day!');
-** </pre></blockquote>
-**
-** This second example is an SQL syntax error.  As a general rule you should
-** always use %q instead of %s when inserting text into a string literal.
-**
-** ^(The %Q option works like %q except it also adds single quotes around
-** the outside of the total string.  Additionally, if the parameter in the
-** argument list is a NULL pointer, %Q substitutes the text "NULL" (without
-** single quotes).)^  So, for example, one could say:
-**
-** <blockquote><pre>
-**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES(%Q)", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-** </pre></blockquote>
-**
-** The code above will render a correct SQL statement in the zSQL
-** variable even if the zText variable is a NULL pointer.
-**
-** ^(The "%w" formatting option is like "%q" except that it expects to
-** be contained within double-quotes instead of single quotes, and it
-** escapes the double-quote character instead of the single-quote
-** character.)^  The "%w" formatting option is intended for safely inserting
-** table and column names into a constructed SQL statement.
-**
-** ^(The "%z" formatting option works like "%s" but with the
-** addition that after the string has been read and copied into
-** the result, [sqlite3_free()] is called on the input string.)^
+** See also:  [built-in printf()], [printf() SQL function]
 */
 char *sqlite3_mprintf(const char*,...);
 char *sqlite3_vmprintf(const char*, va_list);
@@ -4069,6 +4069,9 @@ extern FuncDefHash sqlite3BuiltinFunctions;
 extern int sqlite3PendingByte;
 #endif
 #endif
+#ifdef VDBE_PROFILE
+extern sqlite3_uint64 sqlite3NProfileCnt;
+#endif
 void sqlite3RootPageMoved(sqlite3*, int, int, int);
 void sqlite3Reindex(Parse*, Token*, Token*);
 void sqlite3AlterFunctions(void);
@@ -501,6 +501,12 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double",

 Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);

+#ifdef SQLITE_ENABLE_NULL_TRIM
+  Tcl_SetVar2(interp, "sqlite_options", "null_trim", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "null_trim", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_OMIT_OR_OPTIMIZATION
  Tcl_SetVar2(interp, "sqlite_options", "or_opt", "0", TCL_GLOBAL_ONLY);
 #else
@@ -39,8 +39,8 @@ static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){
  while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
    const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0);
    assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 );
-    if( zSubSql ){
-      assert( zSubSql[0]!='S' );
+    assert( sqlite3_strnicmp(zSubSql,"SELECT",6)!=0 || CORRUPT_DB );
+    if( zSubSql && zSubSql[0]!='S' ){
      rc = execSql(db, pzErrMsg, zSubSql);
      if( rc!=SQLITE_OK ) break;
    }
@@ -648,7 +648,7 @@ int sqlite3VdbeExec(

    assert( pOp>=aOp && pOp<&aOp[p->nOp]);
 #ifdef VDBE_PROFILE
-    start = sqlite3Hwtime();
+    start = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime();
 #endif
    nVmStep++;
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
@@ -7233,7 +7233,7 @@ default: {          /* This is really OP_Noop and OP_Explain */

 #ifdef VDBE_PROFILE
    {
-      u64 endTime = sqlite3Hwtime();
+      u64 endTime = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime();
      if( endTime>start ) pOrigOp->cycles += endTime - start;
      pOrigOp->cnt++;
    }
@@ -554,7 +554,11 @@ struct WalIterator {
 ** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs,
 ** then an SQLite error code is returned and *ppPage is set to 0.
 */
-static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
+static SQLITE_NOINLINE int walIndexPageRealloc(
+  Wal *pWal,               /* The WAL context */
+  int iPage,               /* The page we seek */
+  volatile u32 **ppPage    /* Write the page pointer here */
+){
  int rc = SQLITE_OK;

  /* Enlarge the pWal->apWiData[] array if required */
@@ -573,21 +577,20 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
  }

  /* Request a pointer to the required page from the VFS */
-  if( pWal->apWiData[iPage]==0 ){
-    if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
-      pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ);
-      if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT;
-    }else{
-      rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, 
-          pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
-      );
-      assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 );
-      testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK );
-      if( (rc&0xff)==SQLITE_READONLY ){
-        pWal->readOnly |= WAL_SHM_RDONLY;
-        if( rc==SQLITE_READONLY ){
-          rc = SQLITE_OK;
-        }
+  assert( pWal->apWiData[iPage]==0 );
+  if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
+    pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ);
+    if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT;
+  }else{
+    rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, 
+        pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
+    );
+    assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 );
+    testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK );
+    if( (rc&0xff)==SQLITE_READONLY ){
+      pWal->readOnly |= WAL_SHM_RDONLY;
+      if( rc==SQLITE_READONLY ){
+        rc = SQLITE_OK;
      }
    }
  }
@@ -596,6 +599,16 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
  assert( iPage==0 || *ppPage || rc!=SQLITE_OK );
  return rc;
 }
+static int walIndexPage(
+  Wal *pWal,               /* The WAL context */
+  int iPage,               /* The page we seek */
+  volatile u32 **ppPage    /* Write the page pointer here */
+){
+  if( pWal->nWiData<=iPage || (*ppPage = pWal->apWiData[iPage])==0 ){
+    return walIndexPageRealloc(pWal, iPage, ppPage);
+  }
+  return SQLITE_OK;
+}

 /*
 ** Return a pointer to the WalCkptInfo structure in the wal-index.
@@ -2868,7 +2881,7 @@ int sqlite3WalFindFrame(
  **     table after the current read-transaction had started.
  */
  iMinHash = walFramePage(pWal->minFrame);
-  for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){
+  for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){
    volatile ht_slot *aHash;      /* Pointer to hash table */
    volatile u32 *aPgno;          /* Pointer to array of page numbers */
    u32 iZero;                    /* Frame number corresponding to aPgno[0] */
@@ -2891,6 +2904,7 @@ int sqlite3WalFindFrame(
        return SQLITE_CORRUPT_BKPT;
      }
    }
+    if( iRead ) break;
  }

 #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT
@@ -1395,7 +1395,15 @@ Bitmask sqlite3WhereCodeOneLoopStart(
      if( sqlite3ExprIsVector(pX->pRight) ){
        r1 = rTemp = sqlite3GetTempReg(pParse);
        codeExprOrVector(pParse, pX->pRight, r1, 1);
-        op = aMoveOp[(pX->op - TK_GT) | 0x0001];
+        testcase( pX->op==TK_GT );
+        testcase( pX->op==TK_GE );
+        testcase( pX->op==TK_LT );
+        testcase( pX->op==TK_LE );
+        op = aMoveOp[((pX->op - TK_GT - 1) & 0x3) | 0x1];
+        assert( pX->op!=TK_GT || op==OP_SeekGE );
+        assert( pX->op!=TK_GE || op==OP_SeekGE );
+        assert( pX->op!=TK_LT || op==OP_SeekLE );
+        assert( pX->op!=TK_LE || op==OP_SeekLE );
      }else{
        r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp);
        disableTerm(pLevel, pStart);
@@ -1291,7 +1291,7 @@ static void exprAnalyze(
      exprAnalyze(pSrc, pWC, idxNew);
    }
    pTerm = &pWC->a[idxTerm];
-    pTerm->wtFlags = TERM_CODED|TERM_VIRTUAL;  /* Disable the original */
+    pTerm->wtFlags |= TERM_CODED|TERM_VIRTUAL;  /* Disable the original */
    pTerm->eOperator = 0;
  }

@@ -143,4 +143,18 @@ foreach {tn tcl1 tcl2}  {
  eval $tcl2
 }

+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE zt USING fts4(a, b);
+  INSERT INTO zt(rowid, a, b) VALUES(1, 'unus duo', NULL);
+  INSERT INTO zt(rowid, a, b) VALUES(2, NULL, NULL);
+
+  BEGIN;
+    UPDATE zt SET b='septum' WHERE rowid = 1;
+    UPDATE zt SET b='octo' WHERE rowid = 1;
+  COMMIT;
+
+  SELECT count(*) FROM zt_segdir;
+} {3}
+
+
 finish_test
@@ -22,6 +22,11 @@ ifcapable !offset_sql_func {
  return
 }

+set bNullTrim 0
+ifcapable null_trim {
+  set bNullTrim 1
+}
+
 do_execsql_test func6-100 {
  PRAGMA page_size=4096;
  PRAGMA auto_vacuum=NONE;
@@ -51,6 +56,13 @@ proc loadhex {file} {
 proc hexrecord {args} {
  set hdr ""
  set body ""
+
+  if {$::bNullTrim} {
+    while {[llength $args] && [lindex $args end]=={}} {
+      set args [lrange $args 0 end-1]
+    }
+  }
+
  foreach x $args {
    if {$x==""} {
      append hdr 00
@@ -103,12 +115,17 @@ set F [loadhex test.db]
 db func offrec offrec

 # Test the sanity of the tests.
+if {$bNullTrim} {
+  set offset 8180
+} else {
+  set offset 8179
+}
 do_execsql_test func6-105 {
  SELECT sqlite_offset(d) FROM t1 ORDER BY rowid LIMIT 1;
-} {8179}
+} $offset
 do_test func6-106 {
  set r [hexrecord abc001 1 999 {}]
-  offset_contains_record 8179 $F $r
+  offset_contains_record $offset $F $r
 } 0

 set z100 [string trim [string repeat "0 " 100]]
@@ -148,6 +148,63 @@ do_execsql_test printf2-4.10 {
  SELECT printf('|%,d|%,d|',1234567890,-1234567890);
 } {|1,234,567,890|-1,234,567,890|}

+# 2018-02-19.  Unicode characters with %c
+do_execsql_test printf2-5.100 {
+  SELECT printf('(%8c)',char(11106));
+} {{(       ⭢)}}
+do_execsql_test printf2-5.101 {
+  SELECT printf('(%-8c)',char(11106));
+} {{(⭢       )}}
+do_execsql_test printf2-5.102 {
+  SELECT printf('(%5.3c)',char(1492));
+} {{(  ההה)}}
+do_execsql_test printf2-5.103 {
+  SELECT printf('(%-5.3c)',char(1492));
+} {{(ההה  )}}
+do_execsql_test printf2-5.104 {
+  SELECT printf('(%3.3c)',char(1492));
+} {{(ההה)}}
+do_execsql_test printf2-5.105 {
+  SELECT printf('(%-3.3c)',char(1492));
+} {{(ההה)}}
+do_execsql_test printf2-5.104 {
+  SELECT printf('(%2c)',char(1513));
+} {{( ש)}}
+do_execsql_test printf2-5.106 {
+  SELECT printf('(%-2c)',char(1513));
+} {{(ש )}}
+
+# 2018-02-19.  Unicode characters with the "!" flag in %s and friends.
+do_execsql_test printf2-6.100 {
+  SELECT printf('(%!.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {(הנה)}
+do_execsql_test printf2-6.101 {
+  SELECT printf('(%.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {(הנה)}
+do_execsql_test printf2-6.102 {
+  SELECT printf('(%!5.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(  הנה)}}
+do_execsql_test printf2-6.103 {
+  SELECT printf('(%8.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(  הנה)}}
+do_execsql_test printf2-6.104 {
+  SELECT printf('(%!-5.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(הנה  )}}
+do_execsql_test printf2-6.105 {
+  SELECT printf('(%-8.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(הנה  )}}
+do_execsql_test printf2-6.106 {
+  SELECT printf('(%!.3Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {('הנה')}
+do_execsql_test printf2-6.107 {
+  SELECT printf('(%.6Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {('הנה')}
+do_execsql_test printf2-6.108 {
+  SELECT printf('(%!7.3Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(  'הנה')}}
+do_execsql_test printf2-6.109 {
+  SELECT printf('(%10.6Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(  'הנה')}}


 finish_test
@@ -437,6 +437,113 @@ do_execsql_test 18.6 {
 } {1 1 1 1 2 1}

 
-finish_test
+# 2018-02-13 Ticket https://www.sqlite.org/src/tktview/f484b65f3d6230593c3
+# Incorrect result from a row-value comparison in the WHERE clause.
+#
+do_execsql_test 19.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY,b);
+  INSERT INTO t1(a,b) VALUES(1,11),(2,22),(3,33),(4,44);
+  SELECT * FROM t1 WHERE (a,b)>(0,0) ORDER BY a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.2 {
+  SELECT * FROM t1 WHERE (a,b)>=(0,0) ORDER BY a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.3 {
+  SELECT * FROM t1 WHERE (a,b)<(5,0) ORDER BY a DESC;
+} {4 44 3 33 2 22 1 11}
+do_execsql_test 19.4 {
+  SELECT * FROM t1 WHERE (a,b)<=(5,0) ORDER BY a DESC;
+} {4 44 3 33 2 22 1 11}
+do_execsql_test 19.5 {
+  SELECT * FROM t1 WHERE (a,b)>(3,0) ORDER BY a;
+} {3 33 4 44}
+do_execsql_test 19.6 {
+  SELECT * FROM t1 WHERE (a,b)>=(3,0) ORDER BY a;
+} {3 33 4 44}
+do_execsql_test 19.7 {
+  SELECT * FROM t1 WHERE (a,b)<(3,0) ORDER BY a DESC;
+} {2 22 1 11}
+do_execsql_test 19.8 {
+  SELECT * FROM t1 WHERE (a,b)<=(3,0) ORDER BY a DESC;
+} {2 22 1 11}
+do_execsql_test 19.9 {
+  SELECT * FROM t1 WHERE (a,b)>(3,32) ORDER BY a;
+} {3 33 4 44}
+do_execsql_test 19.10 {
+  SELECT * FROM t1 WHERE (a,b)>(3,33) ORDER BY a;
+} {4 44}
+do_execsql_test 19.11 {
+  SELECT * FROM t1 WHERE (a,b)>=(3,33) ORDER BY a;
+} {3 33 4 44}
+do_execsql_test 19.12 {
+  SELECT * FROM t1 WHERE (a,b)>=(3,34) ORDER BY a;
+} {4 44}
+do_execsql_test 19.13 {
+  SELECT * FROM t1 WHERE (a,b)<(3,34) ORDER BY a DESC;
+} {3 33 2 22 1 11}
+do_execsql_test 19.14 {
+  SELECT * FROM t1 WHERE (a,b)<(3,33) ORDER BY a DESC;
+} {2 22 1 11}
+do_execsql_test 19.15 {
+  SELECT * FROM t1 WHERE (a,b)<=(3,33) ORDER BY a DESC;
+} {3 33 2 22 1 11}
+do_execsql_test 19.16 {
+  SELECT * FROM t1 WHERE (a,b)<=(3,32) ORDER BY a DESC;
+} {2 22 1 11}
+do_execsql_test 19.21 {
+  SELECT * FROM t1 WHERE (0,0)<(a,b) ORDER BY a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.22 {
+  SELECT * FROM t1 WHERE (0,0)<=(a,b) ORDER BY a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.23 {
+  SELECT * FROM t1 WHERE (5,0)>(a,b) ORDER BY a DESC;
+} {4 44 3 33 2 22 1 11}
+do_execsql_test 19.24 {
+  SELECT * FROM t1 WHERE (5,0)>=(a,b) ORDER BY a DESC;
+} {4 44 3 33 2 22 1 11}
+do_execsql_test 19.25 {
+  SELECT * FROM t1 WHERE (3,0)<(a,b) ORDER BY a;
+} {3 33 4 44}
+do_execsql_test 19.26 {
+  SELECT * FROM t1 WHERE (3,0)<=(a,b) ORDER BY a;
+} {3 33 4 44}
+do_execsql_test 19.27 {
+  SELECT * FROM t1 WHERE (3,0)>(a,b) ORDER BY a DESC;
+} {2 22 1 11}
+do_execsql_test 19.28 {
+  SELECT * FROM t1 WHERE (3,0)>=(a,b) ORDER BY a DESC;
+} {2 22 1 11}
+do_execsql_test 19.29 {
+  SELECT * FROM t1 WHERE (3,32)<(a,b) ORDER BY a;
+} {3 33 4 44}
+do_execsql_test 19.30 {
+  SELECT * FROM t1 WHERE (3,33)<(a,b) ORDER BY a;
+} {4 44}
+do_execsql_test 19.31 {
+  SELECT * FROM t1 WHERE (3,33)<=(a,b) ORDER BY a;
+} {3 33 4 44}
+do_execsql_test 19.32 {
+  SELECT * FROM t1 WHERE (3,34)<=(a,b) ORDER BY a;
+} {4 44}
+do_execsql_test 19.33 {
+  SELECT * FROM t1 WHERE (3,34)>(a,b) ORDER BY a DESC;
+} {3 33 2 22 1 11}
+do_execsql_test 19.34 {
+  SELECT * FROM t1 WHERE (3,33)>(a,b) ORDER BY a DESC;
+} {2 22 1 11}
+do_execsql_test 19.35 {
+  SELECT * FROM t1 WHERE (3,33)>=(a,b) ORDER BY a DESC;
+} {3 33 2 22 1 11}
+do_execsql_test 19.36 {
+  SELECT * FROM t1 WHERE (3,32)>=(a,b) ORDER BY a DESC;
+} {2 22 1 11}
+
+# 2018-02-18: Memory leak nexted row-value.  Detected by OSSFuzz.
+#
+do_catchsql_test 20.1 {
+  SELECT 1 WHERE (2,(2,0)) IS (2,(2,0));
+} {0 1}

 finish_test
@@ -279,7 +279,7 @@ ifcapable trace {
  do_tracesql_test 6.2.3 {
    SELECT word, distance FROM t3 WHERE rowid = 10 AND word MATCH 'kiiner';
  } {keener 300
-    {SELECT id, word, rank, k1  FROM "main"."t3_vocab" WHERE langid=0 AND k2>=?1 AND k2<?2}
+    {SELECT id, word, rank, coalesce(k1,word)  FROM "main"."t3_vocab" WHERE langid=0 AND k2>=?1 AND k2<?2}
  }
 }

@@ -0,0 +1,353 @@
+# 2018-02-14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for the editdist3() function in the spellfix extension.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix spellfix4
+
+ifcapable !vtab { finish_test ; return }
+
+load_static_extension db spellfix
+
+do_execsql_test 100 {
+  CREATE TABLE cost1(iLang, cFrom, cTo, iCost);
+  INSERT INTO cost1 VALUES
+    (0, '', '?',  97),
+    (0, '?', '',  98),
+    (0, '?', '?', 99),
+    (0, 'm', 'n', 50),
+    (0, 'n', 'm', 50)
+  ;
+  SELECT editdist3('cost1');
+  SELECT editdist3('anchor','amchor');
+} {{} 50}
+do_execsql_test 110 {
+  SELECT editdist3('anchor','anchoxr');
+} {97}
+do_execsql_test 111 {
+  SELECT editdist3('anchor','xanchor');
+} {97}
+do_execsql_test 112 {
+  SELECT editdist3('anchor','anchorx');
+} {97}
+do_execsql_test 120 {
+  SELECT editdist3('anchor','anchr');
+} {98}
+do_execsql_test 121 {
+  SELECT editdist3('anchor','ancho');
+} {98}
+do_execsql_test 122 {
+  SELECT editdist3('anchor','nchor');
+} {98}
+do_execsql_test 130 {
+  SELECT editdist3('anchor','anchur');
+} {99}
+do_execsql_test 131 {
+  SELECT editdist3('anchor','onchor');
+} {99}
+do_execsql_test 132 {
+  SELECT editdist3('anchor','anchot');
+} {99}
+do_execsql_test 140 {
+  SELECT editdist3('anchor','omchor');
+} {149}
+
+do_execsql_test 200 {
+  INSERT INTO cost1 VALUES
+    (0, 'a', 'ä', 5),
+    (0, 'ss', 'ß', 8)
+  ;
+  SELECT editdist3('cost1');
+  SELECT editdist3('strasse','straße');
+  SELECT editdist3('straße','strasse');
+} {{} 8 196}
+do_execsql_test 210 {
+  SELECT editdist3('baume','bäume');
+} {5}
+do_execsql_test 220 {
+  SELECT editdist3('baum','bäume');
+} {102}
+do_execsql_test 230 {
+  INSERT INTO cost1 VALUES
+    (0, 'ä', 'a', 5),
+    (0, 'ß', 'ss', 8)
+  ;
+  SELECT editdist3('cost1');
+  SELECT editdist3('strasse','straße');
+  SELECT editdist3('straße','strasse');
+} {{} 8 8}
+
+do_execsql_test 300 {
+  DELETE FROM cost1;
+  INSERT INTO cost1 VALUES
+    (0, '', '?',  97),
+    (0, '?', '',  98),
+    (0, '?', '?', 99),
+    (0, 'a', 'e', 50),
+    (0, 'a', 'i', 70),
+    (0, 'a', 'o', 75),
+    (0, 'a', 'u', 81),
+    (0, 'e', 'a', 50),
+    (0, 'e', 'i', 52),
+    (0, 'e', 'o', 72),
+    (0, 'e', 'u', 82),
+    (0, 'i', 'a', 70),
+    (0, 'i', 'e', 52),
+    (0, 'i', 'o', 75),
+    (0, 'i', 'u', 83),
+    (0, 'o', 'a', 75),
+    (0, 'o', 'e', 72),
+    (0, 'o', 'i', 75),
+    (0, 'o', 'u', 40),
+    (0, 'u', 'a', 81),
+    (0, 'u', 'e', 82),
+    (0, 'u', 'i', 83),
+    (0, 'u', 'o', 40),
+    (0, 'm', 'n', 45),
+    (0, 'n', 'm', 45)
+  ;
+  CREATE TABLE words(x TEXT);
+  INSERT INTO words VALUES
+   ('abraham'),
+   ('action'),
+   ('africa'),
+   ('aladdin'),
+   ('alert'),
+   ('alien'),
+   ('amazon'),
+   ('analog'),
+   ('animal'),
+   ('apollo'),
+   ('archive'),
+   ('arnold'),
+   ('aspirin'),
+   ('august'),
+   ('average'),
+   ('bahama'),
+   ('bambino'),
+   ('barcode'),
+   ('bazooka'),
+   ('belgium'),
+   ('between'),
+   ('biology'),
+   ('blonde'),
+   ('border'),
+   ('brave'),
+   ('british'),
+   ('bucket'),
+   ('button'),
+   ('caesar'),
+   ('camilla'),
+   ('cannon'),
+   ('caramel'),
+   ('carpet'),
+   ('catalog'),
+   ('century'),
+   ('chaos'),
+   ('chef'),
+   ('china'),
+   ('circus'),
+   ('classic'),
+   ('clinic'),
+   ('coconut'),
+   ('combine'),
+   ('complex'),
+   ('congo'),
+   ('convert'),
+   ('cosmos'),
+   ('crack'),
+   ('crown'),
+   ('cyclone'),
+   ('deal'),
+   ('delete'),
+   ('denver'),
+   ('detail'),
+   ('diana'),
+   ('direct'),
+   ('dolby'),
+   ('double'),
+   ('dublin'),
+   ('echo'),
+   ('edition'),
+   ('electra'),
+   ('emotion'),
+   ('enjoy'),
+   ('escape'),
+   ('everest'),
+   ('exile'),
+   ('express'),
+   ('family'),
+   ('ferrari'),
+   ('filter'),
+   ('fish'),
+   ('florida'),
+   ('ford'),
+   ('forum'),
+   ('frank'),
+   ('frozen'),
+   ('gallery'),
+   ('garlic'),
+   ('geneva'),
+   ('gibson'),
+   ('gloria'),
+   ('gordon'),
+   ('gravity'),
+   ('ground'),
+   ('habitat'),
+   ('harlem'),
+   ('hazard'),
+   ('herbert'),
+   ('hobby'),
+   ('house'),
+   ('icon'),
+   ('immune'),
+   ('india'),
+   ('inside'),
+   ('isotope'),
+   ('jamaica'),
+   ('jazz'),
+   ('joker'),
+   ('juliet'),
+   ('jupiter'),
+   ('kevin'),
+   ('korea'),
+   ('latin'),
+   ('legal'),
+   ('lexicon'),
+   ('limbo'),
+   ('lithium'),
+   ('logo'),
+   ('lucas'),
+   ('madrid'),
+   ('major'),
+   ('manual'),
+   ('mars'),
+   ('maximum'),
+   ('medical'),
+   ('mental'),
+   ('meter'),
+   ('miguel'),
+   ('mimosa'),
+   ('miranda'),
+   ('modern'),
+   ('money'),
+   ('morgan'),
+   ('motor'),
+   ('mystic'),
+   ('nebula'),
+   ('network'),
+   ('nice'),
+   ('nitro'),
+   ('norway'),
+   ('nurse'),
+   ('octavia'),
+   ('olympic'),
+   ('opus'),
+   ('orient'),
+   ('othello'),
+   ('pacific'),
+   ('panama'),
+   ('paper'),
+   ('parking'),
+   ('pasta'),
+   ('paul'),
+   ('people'),
+   ('permit'),
+   ('phrase'),
+   ('pilgrim'),
+   ('planet'),
+   ('pocket'),
+   ('police'),
+   ('popular'),
+   ('prefer'),
+   ('presto'),
+   ('private'),
+   ('project'),
+   ('proxy'),
+   ('python'),
+   ('quota'),
+   ('rainbow'),
+   ('raymond'),
+   ('region'),
+   ('report'),
+   ('reward'),
+   ('risk'),
+   ('robot'),
+   ('rose'),
+   ('russian'),
+   ('sailor'),
+   ('salt'),
+   ('saturn'),
+   ('scorpio'),
+   ('second'),
+   ('seminar'),
+   ('shadow'),
+   ('shave'),
+   ('shock'),
+   ('silence'),
+   ('sinatra'),
+   ('sleep'),
+   ('social'),
+   ('sonata'),
+   ('spain'),
+   ('sphere'),
+   ('spray'),
+   ('state'),
+   ('stone'),
+   ('strong'),
+   ('sugar'),
+   ('supreme'),
+   ('swing'),
+   ('talent'),
+   ('telecom'),
+   ('thermos'),
+   ('tina'),
+   ('tommy'),
+   ('torso'),
+   ('trade'),
+   ('trick'),
+   ('tropic'),
+   ('turtle'),
+   ('uniform'),
+   ('user'),
+   ('vega'),
+   ('vertigo'),
+   ('village'),
+   ('visible'),
+   ('vocal'),
+   ('voyage'),
+   ('weekend'),
+   ('winter'),
+   ('year'),
+   ('zipper')
+  ;
+  SELECT editdist3('cost1');
+} {{}}
+do_execsql_test 310 {
+  SELECT editdist3(a.x,b.x), a.x, b.x
+    FROM words a, words b
+   WHERE a.x<b.x
+   ORDER BY 1, 2
+   LIMIT 20
+} {139 bucket pocket 144 meter motor 149 manual mental 169 crack trick 173 sinatra sonata 174 edition emotion 174 major motor 174 risk rose 174 state stone 194 deal detail 196 alert talent 196 analog catalog 196 deal legal 196 ford forum 196 risk trick 196 stone strong 197 china tina 197 congo logo 197 diana tina 197 florida gloria}
+do_execsql_test 320 {
+  SELECT md5sum(ed||'/'||sx||'/'||sy||',') FROM (
+      SELECT editdist3(a.x,b.x) AS ed, a.x AS sx, b.x AS sy
+        FROM words a, words b
+       WHERE a.x<b.x
+       ORDER BY 1, 2
+  )
+} {69d0a31872203a775e19325ea98cd053}
+
+finish_test
@@ -0,0 +1,52 @@
+# 2018-02-15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the WITH clause in TRIGGERs and VIEWs.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix with4
+
+ifcapable {!cte} {
+  finish_test
+  return
+}
+
+do_execsql_test 100 {
+  ATTACH ':memory:' AS aux;
+  CREATE TABLE main.t1(a,b);
+  CREATE TABLE aux.t2(x,y);
+  INSERT INTO t1 VALUES(1,2);
+  INSERT INTO t2 VALUES(3,4);
+} {}
+do_catchsql_test 110 {
+  CREATE VIEW v1 AS SELECT * FROM t1, aux.t2;
+} {1 {view v1 cannot reference objects in database aux}}
+do_catchsql_test 120 {
+  CREATE VIEW v2 AS WITH v(m,n) AS (SELECT x,y FROM aux.t2) SELECT * FROM t1, v;
+} {1 {view v2 cannot reference objects in database aux}}
+do_catchsql_test 130 {
+  CREATE VIEW v2 AS WITH v(m,n) AS (SELECT 5,?2) SELECT * FROM t1, v;
+} {1 {parameters are not allowed in views}}
+
+do_catchsql_test 200 {
+  CREATE TRIGGER r1 AFTER INSERT ON t1 BEGIN
+     WITH v(m,n) AS (SELECT x,y FROM aux.t2) SELECT * FROM t1, v;
+  END;
+} {1 {trigger r1 cannot reference objects in database aux}}
+do_catchsql_test 210 {
+  CREATE TRIGGER r1 AFTER INSERT ON t1 BEGIN
+     WITH v(m,n) AS (SELECT 5,?2) SELECT * FROM t1, v;
+  END;
+} {1 {trigger cannot use variables}}
+
+finish_test
@@ -39,6 +39,8 @@ LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA"
 BASELINE="trunk"
 doExplain=0
 doCachegrind=1
+doVdbeProfile=0
+doWal=1
 while test "$1" != ""; do
  case $1 in
    --reprepare)
@@ -62,8 +64,11 @@ while test "$1" != ""; do
    --temp)
        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6"
        ;;
+    --legacy)
+	doWal=0
+        ;;
    --wal)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
+        doWal=1
        ;;
    --size)
        shift; SIZE=$1
@@ -78,6 +83,7 @@ while test "$1" != ""; do
        rm -f vdbe_profile.out
        CC_OPTS="$CC_OPTS -DVDBE_PROFILE"
        doCachegrind=0
+        doVdbeProfile=1
        ;;
    --lean)
        CC_OPTS="$CC_OPTS $LEAN_OPTS"
@@ -132,11 +138,17 @@ while test "$1" != ""; do
  esac
  shift
 done
+if test $doWal -eq 1; then
+  SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
+fi
 SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE"
 echo "NAME           = $NAME" | tee summary-$NAME.txt
 echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
 echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
 rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
+if test $doVdbeProfile -eq 1; then
+  rm -f vdbe_profile.out
+fi
 $CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
 size sqlite3.o | tee -a summary-$NAME.txt
 if test $doExplain -eq 1; then
@@ -163,6 +175,10 @@ fi
 if test $doExplain -eq 1; then
  ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
 fi
-if test "$NAME" != "$BASELINE"; then
+if test $doVdbeProfile -eq 1; then
+  tclsh ../sqlite/tool/vdbe_profile.tcl >vdbeprofile-$NAME.txt
+  open vdbeprofile-$NAME.txt
+fi
+if test "$NAME" != "$BASELINE" -a $doVdbeProfile -ne 1; then
  fossil test-diff --tk -c 20 cout-$BASELINE.txt cout-$NAME.txt
 fi
Author	SHA1	Message	Date
dan	4a0baed049	Update zonefile README.md file to mention the frame cache. FossilOrigin-Name: 84e9351bbd0a9bf9e1d64a441366a90276c231b44548cbc947f7fb2842f82490	2018-02-27 20:09:45 +00:00
dan	f736acd40c	Rationalize some code in zonefile.c. Fix other minor issues in the same. FossilOrigin-Name: f11beb16a87cc967e896cf67121b1e4045e427ebdc6a424e9f009ffced955d36	2018-02-27 19:50:28 +00:00
mistachkin	d276b00554	Enhance test 'zonefile1-6.5' to account for platform differences. FossilOrigin-Name: 8b6178403fb6a7f87502c42026caf3da1c355b2ed9e780b347f4a05d7ffc1e18	2018-02-27 15:47:41 +00:00
mistachkin	fd9a916c4a	Adjustments to test numbers for the 'zonefile' extension. FossilOrigin-Name: 55de6f14d49342394397c0dc29c1e98c927ef99d57ec1f71c79099f584be20d1	2018-02-27 15:42:29 +00:00
dan	4ec163f7ae	Have the zonefile extension use binary instead of text keys. FossilOrigin-Name: 39a4267fc9cec77fd8d9be25c73b848e77a68906253cc75d61fe90e549bafa27	2018-02-27 14:26:33 +00:00
dan	6c3e64e653	Add extra parameter to zonefileCodecCreate() to indicate whether the new object will be used for mock-encryption or mock-decryption. FossilOrigin-Name: 231832c4cb15862e61dfcc00fba9ab78ca7e2442a0d4aa1a98a191f5f8b4cff3	2018-02-26 07:58:39 +00:00
dan	87f1670de8	Test edge cases in the zonefile module. Fix a broken error message in the same. FossilOrigin-Name: 1764ade22b52eba0226ae2e6e837a1b0967023eabd7d50e9f87c5e7042ea2f12	2018-02-24 08:26:21 +00:00
dan	e68ffadbf4	Fix a problem with handling "k >= ?" constraints in the zonefile module. FossilOrigin-Name: 9a99afafa3fb9ec4823e5030cb04a685d64942ec04a1154b729dc4c0f313dc4a	2018-02-23 21:01:15 +00:00
dan	9f908122af	Add test cases and minor fixes for the zonefile module. FossilOrigin-Name: dbbcbf0066ef82c789981feff78d95861a836e6e23a22ad3bb698dd257062f75	2018-02-23 19:21:19 +00:00
dan	2e2bc60300	Fix a problem causing SQLITE_HAVE_LZ4 builds of zonefile to fail unless SQLITE_HAVE_ZLIB is also defined. FossilOrigin-Name: 994aa77d7826ab220727595a6a21f1f95a8f7590798879706016665ca1934dd1	2018-02-23 14:09:59 +00:00
dan	d9cad97389	Fix a problem with DROP TABLE statements on zonefile virtual tables. FossilOrigin-Name: d2ed6910b86a086911d3c0670c9778e222186d8733fc81bcc081932d70e0452a	2018-02-23 14:02:37 +00:00
mistachkin	7f837b66fa	Fix harmless compiler warnings in the zonefile extension seen with MSVC. FossilOrigin-Name: d28003941ceca7fb707fcdb09bf8b00a6c1b728956e018d7ecfb6852d7dd469b	2018-02-23 14:00:38 +00:00
dan	241676abcc	Avoid running a test case that requires zstd in non-SQLITE_HAVE_ZSTD builds. FossilOrigin-Name: d716dff4441b4d28e0184ce06508bc9122451103ead8edbfbf3c87b666445ee6	2018-02-23 13:58:17 +00:00
mistachkin	c0905baac8	Merge updates from trunk. FossilOrigin-Name: 53f2100a296f64d206f1caf2d754820a7898bb33b31a6ab2169957d228d5d2a0	2018-02-23 13:45:05 +00:00
mistachkin	5facffbc70	Fix harmless compiler warnings in the zipfile extension seen with MSVC. FossilOrigin-Name: 15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414	2018-02-23 13:38:54 +00:00
mistachkin	fda0867873	Enable the JSON1 extension when compiling the test fixture with MSVC. FossilOrigin-Name: e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4	2018-02-23 13:38:20 +00:00
dan	563d3242d7	Add tests cases and fix some minor zonefile problems. FossilOrigin-Name: f4d42162fa2196078cea4db9c2247a08f5439d29e49f5e8dc23db317b8101865	2018-02-22 21:06:08 +00:00
drh	4f46b58a2d	For the "SQLite archive" link in the README.md file, point to the CLI .archive command documentation, not the original "sqlar" website. FossilOrigin-Name: 4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7	2018-02-22 18:30:53 +00:00
drh	d6ec7a6ab3	Updates to the README.md file. Mention the ability to use SQLite-archives for download. FossilOrigin-Name: ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add	2018-02-22 18:28:19 +00:00
dan	2707b56334	Add an LRU cache of uncompressed frame content to the zonefile virtual table implementation. FossilOrigin-Name: 883e7e75d65622e8d06c46e48b7cc756cc0e3345b8124a8038cab0a8d51d0458	2018-02-22 16:46:42 +00:00
dan	86a1bf4815	Modifications to the zonefile module to make it easier to add a cache of uncompressed frame content. FossilOrigin-Name: d9d5cc62f11058f9ba560381367ff4765dbbde08184e55abdb50ae1b6bf4a016	2018-02-21 21:15:45 +00:00
dan	d98b4b24f2	Have zonefile store encryption keys in a hash-table instead of a linked list. Add extra tests for key management. FossilOrigin-Name: 3a63ea652546a4c63eccd72665becff38a97a0e39d2f11703cb6899451570fd4	2018-02-21 16:36:08 +00:00
dan	92c01308cc	In zonefile, change the "file TEXT" column back to "fileid INTEGER". The fileid can be used as a key with the associated zonefile_files table, which contains more information than just the filename. FossilOrigin-Name: 38d23888cf5a7117c51bd9211bd93ec52a30360f7eb2bc83a13910c5d85fe739	2018-02-21 10:43:19 +00:00
drh	c0ec2f7762	Remove an unreachable branch in the walIndexPage() logic. FossilOrigin-Name: 4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b	2018-02-21 01:48:22 +00:00
drh	8d3e15eec4	Small performance improvement in sqltie3WalFindFrame(). FossilOrigin-Name: 52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350	2018-02-21 01:05:37 +00:00
drh	4f4e49a98b	Remove an unnecessary memset(). FossilOrigin-Name: e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893	2018-02-21 00:48:02 +00:00
drh	2e178d7321	Make the walIndexPage() routine about 3x faster by factoring out the seldom used reallocation logic into a separate subroutine. FossilOrigin-Name: e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b	2018-02-20 22:20:57 +00:00
dan	cc6cece39a	Update README.md in ext/zonefile. FossilOrigin-Name: a13b2b38137025d04bbfc1b12f0d0563dcf7e0ab1d4b06ffc7cacf994f00e188	2018-02-20 20:57:20 +00:00
dan	92c7c98ac4	Fix a problem with reading the "file" column of a zonefile virtual table. FossilOrigin-Name: d8d0bdcb405610978393bb8f3ef30ee868fddda0d129036f8d0102012fe0d5d4	2018-02-20 19:25:49 +00:00
dan	b0e7fff492	Instead of just the frame number, store frame sizes and offsets in zonefile shadow table %_shadow_idx. FossilOrigin-Name: 56801c461c5d19cf96146fe0fa7f725c81da5cd7495c9608cd044125d00fecfe	2018-02-20 18:47:24 +00:00
drh	7436f1e491	In the speed-check.sh script, make WAL mode the default. Add the --legacy option to do DELETE mode. Continue to use DELETE mode for long-term performance graphs since WAL mode was not always available, but use WAL mode for day-to-day performance measurements. FossilOrigin-Name: cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd	2018-02-20 17:02:31 +00:00
drh	3993678d3d	Remove a superfluous "#if 1". No logic changes. FossilOrigin-Name: caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c	2018-02-20 16:39:11 +00:00
drh	cc398969e0	Optimize calls to sqlite3_mprintf("%z...") so that they attempt to append text onto the end of the existing memory allocation rather than reallocating and copying. FossilOrigin-Name: 4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda	2018-02-20 15:23:37 +00:00
drh	b0b6f8783c	Remove a lot of the text describing extended format options from the documentation on sqlite3_mprintf() and friends, since that information is now covered by the separate printf.html document. Provide links to that other document. No changes to code. FossilOrigin-Name: 99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb	2018-02-20 13:46:20 +00:00
drh	57a93fa38c	Enhance the string formatter (used by printf()) so that the width and precision of string substitution operators refer to characters instead of bytes when the alternate-form-2 flag ("!") is used. Also fix the %c substition to always work within unicode, regardless of the alternate-form-2 flag. FossilOrigin-Name: c883c4d33f4cd722982aca17eeb355d19eeed793991d923b5494d173b1d0c73a	2018-02-19 22:46:19 +00:00
drh	a15a7c352c	Printing a value of 0 using %c terminates the string. FossilOrigin-Name: 255612f0a131f2f522cbca3cc5a1edcf7e38938abca25ba421e7e38a422db9c9	2018-02-19 21:58:16 +00:00
dan	ccc9acc059	Add support for invoking encryption hooks to zonefile. And mock encryption method "xor" for testing. FossilOrigin-Name: 55cf920c5a13473d04f0cb885117c04b2bc054bfed6ee549be84cb9485c104d2	2018-02-19 21:07:20 +00:00
drh	57f29b7bfb	More test cases for unicode handling in printf. FossilOrigin-Name: ac188211119bfd77f43e4aba8acab733a56aac90b4c8a11f7ef8e6388582438f	2018-02-19 20:41:01 +00:00
drh	cf7c8370d1	Test cases and a bug fix on the new unicode handling in %c. FossilOrigin-Name: e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab	2018-02-19 20:23:20 +00:00
drh	136102beda	Fix the %c substitution in printf() so that it works with unicode characters. FossilOrigin-Name: c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0	2018-02-19 18:56:52 +00:00
drh	57e3ba7623	Make the alternate-form-2 flag ("!") change the meaning of width and precision from bytes to characters for the %q, %Q, and %w extensions of printf(). FossilOrigin-Name: 391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c	2018-02-19 18:03:10 +00:00
drh	62856465f2	In the printf() library, measure width and precision in characters rather than bytes if the "!" (alternate-form-2) flag is present on a %s or %z substitution. FossilOrigin-Name: ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636	2018-02-19 17:03:23 +00:00
drh	26f625fbf4	Only try to use the geteuid() interface on unix if HAVE_FCHOWN is defined. This fixes the build for vxWorks, we are told. FossilOrigin-Name: 38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e	2018-02-19 16:34:31 +00:00
dan	cbce846ea5	Add support for the ExtendedHeaderSize header field to zonefile. FossilOrigin-Name: 78267a091307e2c29a4fb1606fa9c79939fe010b801749614f4c48dc8715810e	2018-02-19 16:28:42 +00:00
dan	c65faba468	Modify the zonefile format in order to avoid depending on the filesize to determine the extent of the final frame. See README.md for details. FossilOrigin-Name: 4dbe0cba3fad9a752834d795127cf35eed21fab63b18a48f75d5c1e96ca77447	2018-02-19 14:27:24 +00:00
drh	a224ee2a1b	Fix an assert so that it compares two CellInfo objects field by field instead of using memcmp(). Memcmp() does not work on x86 because of uninitialized padding bytes. FossilOrigin-Name: 88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477	2018-02-19 13:53:56 +00:00
drh	e28eb64d09	Fix a memory leak in the processing of nested row values. This problem has existed every since row values support was added (version 3.15.0, 2016-10-14) but was only just now detected by OSSFuzz. FossilOrigin-Name: 2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de	2018-02-18 17:50:03 +00:00
mistachkin	091881bbb0	Port mutex enhancments from check-in [f53b8a573b] to the Win32 implementation. FossilOrigin-Name: 74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045	2018-02-18 00:54:06 +00:00
dan	a88b05361c	Add support for "brotli" compression to the zonefile module. FossilOrigin-Name: 3eb25b3fa5733b4418e7e2633be34b763e2c70342bb9c418a07c9f7d4b196fac	2018-02-17 20:22:23 +00:00
dan	aac12cb868	Add support for compression types "lz4" and "lz4hc" to the zonefile module. FossilOrigin-Name: bbe5b21ffab3cd312680ca9f179c5847790c17fb91d4174985153c6c398d48e3	2018-02-17 19:38:02 +00:00
dan	19e6b8f496	Add support for compression methods "zstd" and "zstd_global_dict". FossilOrigin-Name: a993a50bb8d5a3bf7cf79e09204814e172ba0bf9b3949e81912ef83f0d4bb44e	2018-02-17 18:33:43 +00:00
dan	c551550808	Fix a data race causing a tsan complaint with SQLITE_ENABLE_API_ARMOR builds on unix. The race condition is not dangerous in practice, it just upsets tsan. FossilOrigin-Name: f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3	2018-02-17 07:38:50 +00:00
drh	713f34ace9	Do not allow parameters or schema references inside of WITH clause of triggers and views. This fixes a bug discovered by OSSFuzz and present since common-table-expressions were first added in 2014-02-03. FossilOrigin-Name: b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51	2018-02-15 21:00:37 +00:00
dan	21f433e821	Add support for zlib compression to the zonefile module. FossilOrigin-Name: 72b8a7ef98d84460718378b9d17477599df39b4216015f8967674dd02b54b406	2018-02-15 20:37:58 +00:00
drh	f4bc6c43d7	Improve performance of editdist3() by keeping the costs in sorted order. Also add a new regression test to editdist3(). FossilOrigin-Name: dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256	2018-02-15 03:56:33 +00:00
drh	46e835a2c4	Reduce the number of calls to strncmp() required to run editDist3Core(). FossilOrigin-Name: afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a	2018-02-15 03:05:56 +00:00
drh	501ea05630	Fix the --heap, --pagecache, --lookaside and other options to the command-line shell that invoke sqlite3_config() so that they work again. FossilOrigin-Name: 00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e	2018-02-15 01:03:37 +00:00
drh	ee269a60c9	Add the --readonly option to the ".open" command in the CLI. FossilOrigin-Name: 06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076	2018-02-14 23:27:43 +00:00
drh	d9274a8ae3	Performance optimizations to the editdist3() function in the spellfix extension. FossilOrigin-Name: 70d304dcbac4c3fd5e3b96108bffea2ce6c0db19c847397d5c5e268bb90a981d	2018-02-14 20:58:36 +00:00
drh	0fae06fcb6	Disable assert() in the spellfix extension if not compiled with SQLITE_DEBUG. FossilOrigin-Name: 3c53ee0fdea4cbf2590e2b289b021b0ef8b7ead2945db1bdfc767432d9447acb	2018-02-14 20:25:33 +00:00
drh	4e6e5cef6c	New test cases for the editdist3() function in the spellfix extension. FossilOrigin-Name: 2d3f1f22d53034576a0ec8d9c360fb19efbb05b2e95ad1b09d120fb1b7da33fd	2018-02-14 15:30:48 +00:00
drh	e2d27e0259	Try to optimize spellfix1 by storing a NULL in the k1 column of %_vocab if it would otherwise have the same value as the word column. FossilOrigin-Name: b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2	2018-02-14 14:13:09 +00:00
drh	a9abfb1e6c	Improved test cases for the [f484b65f3d6230593c34f] bug. FossilOrigin-Name: 1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289	2018-02-13 19:13:05 +00:00
drh	4d1c6845ff	Fix an incorrect table lookup used to find the appropriate search operator for a WHERE clause on a row-value inequality. The incorrect table lookup was causing an incorrect answer for the less-than operator. Fix for ticket [f484b65f3d6230593c34f11] FossilOrigin-Name: f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336	2018-02-13 18:48:08 +00:00
drh	d446e79545	Remove an unused variable from FTS4. FossilOrigin-Name: 27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2	2018-02-13 16:30:19 +00:00
dan	e84c498be1	Ensure that multiple updates of the same FTS4 row (i.e. the row with the same rowid) within a single transaction are written to separate segments. Fix for [d6ec09ec]. FossilOrigin-Name: d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64	2018-02-13 16:21:32 +00:00
drh	c083cb4422	Updates to the speed-check.sh script for improvements to the --vdbeprofile option. FossilOrigin-Name: f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1	2018-02-12 20:50:28 +00:00
drh	35043cc32a	When compiling with VDBE_PROFILE, add the sqlite3NProfileCnt global variable which can be used to measure per-opcode values other than elapse time, if non-zero. FossilOrigin-Name: d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30	2018-02-12 20:27:34 +00:00
dan	1db7db1f03	Update test file func6.test so that it works with SQLITE_ENABLE_NULL_TRIM builds. FossilOrigin-Name: e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc	2018-02-12 15:27:32 +00:00
drh	00c3ef2df9	Fix a possible infinite loop in VACUUM that can come up when the schema has been carefully corrupted. Problem discovered by OSSFuzz. Test cases in TH3. FossilOrigin-Name: 27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2	2018-02-12 13:30:34 +00:00