Add tests for structured-exception-handling and wal2. Fix a problem with the same.

FossilOrigin-Name: 87242777ec869d698c5c78283b98677105e8b5a8b0d8b71a696b8460d4303783

Makefile.in

@@ -425,6 +425,7 @@ TESTSRC += \
   $(TOP)/ext/expert/sqlite3expert.c \
   $(TOP)/ext/expert/test_expert.c \
   $(TOP)/ext/misc/amatch.c \
+  $(TOP)/ext/misc/bgckpt.c \
   $(TOP)/ext/misc/appendvfs.c \
   $(TOP)/ext/misc/basexx.c \
   $(TOP)/ext/misc/carray.c \

Makefile.linux-gcc

@@ -22,12 +22,6 @@ TOP = ../sqlite
 BCC = gcc -g -O0
 #BCC = /opt/ancic/bin/c89 -0
 
-#### If the target operating system supports the "usleep()" system
-#    call, then define the HAVE_USLEEP macro for all C modules.
-#
-#USLEEP = 
-USLEEP = -DHAVE_USLEEP=1
-
 #### If you want the SQLite library to be safe for use within a 
 #    multi-threaded program, then define the following macro
 #    appropriately:

Makefile.msc

@@ -52,6 +52,13 @@ MINIMAL_AMALGAMATION = 0
 USE_STDCALL = 0
 !ENDIF
 
+# Set this non-0 to use structured exception handling (SEH) for WAL mode
+# in the core library.
+#
+!IFNDEF USE_SEH
+USE_SEH = 0
+!ENDIF
+
 # Set this non-0 to have the shell executable link against the core dynamic
 # link library.
 #
@@ -389,6 +396,13 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
 !ENDIF
 
+# Should structured exception handling (SEH) be enabled for WAL mode in
+# the core library?
+#
+!IF $(USE_SEH)!=0
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_USE_SEH=1
+!ENDIF
+
 # These are the "extended" SQLite compilation options used when compiling for
 # the Windows 10 platform.
 #
@@ -1536,6 +1550,7 @@ TESTEXT = \
   $(TOP)\ext\misc\amatch.c \
   $(TOP)\ext\misc\appendvfs.c \
   $(TOP)\ext\misc\basexx.c \
+  $(TOP)\ext\misc\bgckpt.c \
   $(TOP)\ext\misc\carray.c \
   $(TOP)\ext\misc\cksumvfs.c \
   $(TOP)\ext\misc\closure.c \

autoconf/Makefile.msc

@@ -52,6 +52,13 @@ MINIMAL_AMALGAMATION = 0
 USE_STDCALL = 0
 !ENDIF
 
+# Set this non-0 to use structured exception handling (SEH) for WAL mode
+# in the core library.
+#
+!IFNDEF USE_SEH
+USE_SEH = 0
+!ENDIF
+
 # Set this non-0 to have the shell executable link against the core dynamic
 # link library.
 #
@@ -311,6 +318,13 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
 !ENDIF
 
+# Should structured exception handling (SEH) be enabled for WAL mode in
+# the core library?
+#
+!IF $(USE_SEH)!=0
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_USE_SEH=1
+!ENDIF
+
 # These are the "extended" SQLite compilation options used when compiling for
 # the Windows 10 platform.
 #

doc/wal2.md

@@ -0,0 +1,98 @@
+
+Wal2 Mode Notes
+===============
+
+## Activating/Deactivating Wal2 Mode
+
+"Wal2" mode is very similar to "wal" mode. To change a database to wal2 mode,
+use the command:
+
+>
+     PRAGMA journal_mode = wal2;
+
+It is not possible to change a database directly from "wal" mode to "wal2"
+mode. Instead, it must first be changed to rollback mode. So, to change a wal
+mode database to wal2 mode, the following two commands may be used:
+
+>
+     PRAGMA journal_mode = delete;
+     PRAGMA journal_mode = wal2;
+
+A database in wal2 mode may only be accessed by versions of SQLite compiled
+from this branch. Attempting to use any other version of SQLite results in an
+SQLITE_NOTADB error. A wal2 mode database may be changed back to rollback mode
+(making it accessible by all versions of SQLite) using:
+
+>
+     PRAGMA journal_mode = delete;
+
+## The Advantage of Wal2 Mode
+
+In legacy wal mode, when a writer writes data to the database, it doesn't
+modify the database file directly. Instead, it appends new data to the
+"<database>-wal" file. Readers read data from both the original database
+file and the "<database>-wal" file. At some point, data is copied from the
+"<database>-wal" file into the database file, after which the wal file can
+be deleted or overwritten. Copying data from the wal file into the database
+file is called a "checkpoint", and may be done explictly (either by "PRAGMA
+wal_checkpoint" or sqlite3_wal_checkpoint_v2()), or
+automatically (by configuring "PRAGMA wal_autocheckpoint" - this is the
+default).
+
+Checkpointers do not block writers, and writers do not block checkpointers.
+However, if a writer writes to the database while a checkpoint is ongoing,
+then the new data is appended to the end of the wal file. This means that,
+even following the checkpoint, the wal file cannot be overwritten or deleted,
+and so all subsequent transactions must also be appended to the wal file. The
+work of the checkpointer is not wasted - SQLite remembers which parts of the
+wal file have already been copied into the db file so that the next checkpoint
+does not have to do so again - but it does mean that the wal file may grow
+indefinitely if the checkpointer never gets a chance to finish without a
+writer appending to the wal file. There are also circumstances in which
+long-running readers may prevent a checkpointer from checkpointing the entire
+wal file - also causing the wal file to grow indefinitely in a busy system.
+
+Wal2 mode does not have this problem. In wal2 mode, wal files do not grow
+indefinitely even if the checkpointer never has a chance to finish
+uninterrupted.
+
+In wal2 mode, the system uses two wal files instead of one. The files are named
+"<database>-wal" and "<database>-wal2", where "<database>" is of
+course the name of the database file. When data is written to the database, the
+writer begins by appending the new data to the first wal file. Once the first
+wal file has grown large enough, writers switch to appending data to the second
+wal file. At this point the first wal file can be checkpointed (after which it
+can be overwritten). Then, once the second wal file has grown large enough and
+the first wal file has been checkpointed, writers switch back to the first wal
+file. And so on.
+
+## Application Programming
+
+From the point of view of the user, the main differences between wal and 
+wal2 mode are to do with checkpointing:
+
+  * In wal mode, a checkpoint may be attempted at any time. In wal2 
+    mode, the checkpointer has to wait until writers have switched 
+    to the "other" wal file before a checkpoint can take place.
+
+  * In wal mode, the wal-hook (callback registered using
+    sqlite3_wal_hook()) is invoked after a transaction is committed
+    with the total number of pages in the wal file as an argument. In wal2
+    mode, the argument is either the total number of uncheckpointed pages in
+    both wal files, or - if the "other" wal file is empty or already
+    checkpointed - 0.
+
+Clients are recommended to use the same strategies for checkpointing wal2 mode
+databases as for wal databases - by registering a wal-hook using
+sqlite3_wal_hook() and attempting a checkpoint when the parameter
+exceeds a certain threshold.
+
+However, it should be noted that although the wal-hook is invoked after each
+transaction is committed to disk and database locks released, it is still
+invoked from within the sqlite3_step() call used to execute the "COMMIT"
+command. In BEGIN CONCURRENT systems, where the "COMMIT" is often protected by
+an application mutex, this may reduce concurrency. In such systems, instead of
+executing a checkpoint from within the wal-hook, a thread might defer this
+action until after the application mutex has been released.
+
+

ext/fts5/fts5.h

@@ -263,7 +263,7 @@ struct Fts5PhraseIter {
 **   See xPhraseFirstColumn above.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 3 */
+  int iVersion;                   /* Currently always set to 2 */
 
   void *(*xUserData)(Fts5Context*);
 
@@ -541,7 +541,7 @@ struct fts5_api {
   int (*xCreateTokenizer)(
     fts5_api *pApi,
     const char *zName,
-    void *pContext,
+    void *pUserData,
     fts5_tokenizer *pTokenizer,
     void (*xDestroy)(void*)
   );
@@ -550,7 +550,7 @@ struct fts5_api {
   int (*xFindTokenizer)(
     fts5_api *pApi,
     const char *zName,
-    void **ppContext,
+    void **ppUserData,
     fts5_tokenizer *pTokenizer
   );
 
@@ -558,7 +558,7 @@ struct fts5_api {
   int (*xCreateFunction)(
     fts5_api *pApi,
     const char *zName,
-    void *pContext,
+    void *pUserData,
     fts5_extension_function xFunction,
     void (*xDestroy)(void*)
   );

ext/fts5/fts5Int.h

@@ -154,6 +154,10 @@ typedef struct Fts5Config Fts5Config;
 **   attempt to merge together. A value of 1 sets the object to use the 
 **   compile time default. Zero disables auto-merge altogether.
 **
+** bContentlessDelete:
+**   True if the contentless_delete option was present in the CREATE 
+**   VIRTUAL TABLE statement.
+**
 ** zContent:
 **
 ** zContentRowid:
@@ -188,6 +192,7 @@ struct Fts5Config {
   int nPrefix;                    /* Number of prefix indexes */
   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
   int eContent;                   /* An FTS5_CONTENT value */
+  int bContentlessDelete;         /* "contentless_delete=" option (dflt==0) */
   char *zContent;                 /* content table */ 
   char *zContentRowid;            /* "content_rowid=" option value */ 
   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
@@ -209,6 +214,7 @@ struct Fts5Config {
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
   int bSecureDelete;              /* 'secure-delete' */
+  int nDeleteMerge;           /* 'deletemerge' */
 
   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   char **pzErrmsg;
@@ -531,6 +537,9 @@ int sqlite3Fts5IndexReset(Fts5Index *p);
 
 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 
+int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
+int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
+
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
@@ -615,6 +624,11 @@ int sqlite3Fts5HashWrite(
 */
 void sqlite3Fts5HashClear(Fts5Hash*);
 
+/*
+** Return true if the hash is empty, false otherwise.
+*/
+int sqlite3Fts5HashIsEmpty(Fts5Hash*);
+
 int sqlite3Fts5HashQuery(
   Fts5Hash*,                      /* Hash table to query */
   int nPre,
@@ -636,6 +650,7 @@ void sqlite3Fts5HashScanEntry(Fts5Hash *,
 );
 
 
+
 /*
 ** End of interface to code in fts5_hash.c.
 **************************************************************************/

ext/fts5/fts5_config.c

@@ -22,6 +22,8 @@
 #define FTS5_DEFAULT_CRISISMERGE   16
 #define FTS5_DEFAULT_HASHSIZE    (1024*1024)
 
+#define FTS5_DEFAULT_DELETE_AUTOMERGE 10      /* default 10% */
+
 /* Maximum allowed page size */
 #define FTS5_MAX_PAGE_SIZE (64*1024)
 
@@ -352,6 +354,16 @@ static int fts5ConfigParseSpecial(
     return rc;
   }
 
+  if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bContentlessDelete = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
   if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
     if( pConfig->zContentRowid ){
       *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
@@ -596,6 +608,28 @@ int sqlite3Fts5ConfigParse(
     sqlite3_free(zTwo);
   }
 
+  /* We only allow contentless_delete=1 if the table is indeed contentless. */
+  if( rc==SQLITE_OK 
+   && pRet->bContentlessDelete 
+   && pRet->eContent!=FTS5_CONTENT_NONE 
+  ){
+    *pzErr = sqlite3_mprintf(
+        "contentless_delete=1 requires a contentless table"
+    );
+    rc = SQLITE_ERROR;
+  }
+
+  /* We only allow contentless_delete=1 if columnsize=0 is not present. 
+  **
+  ** This restriction may be removed at some point. 
+  */
+  if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){
+    *pzErr = sqlite3_mprintf(
+        "contentless_delete=1 is incompatible with columnsize=0"
+    );
+    rc = SQLITE_ERROR;
+  }
+
   /* If a tokenizer= option was successfully parsed, the tokenizer has
   ** already been allocated. Otherwise, allocate an instance of the default
   ** tokenizer (unicode61) now.  */
@@ -890,6 +924,18 @@ int sqlite3Fts5ConfigSetValue(
     }
   }
 
+  else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){
+    int nVal = -1;
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      nVal = sqlite3_value_int(pVal);
+    }else{
+      *pbBadkey = 1;
+    }
+    if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE;
+    if( nVal>100 ) nVal = 0;
+    pConfig->nDeleteMerge = nVal;
+  }
+
   else if( 0==sqlite3_stricmp(zKey, "rank") ){
     const char *zIn = (const char*)sqlite3_value_text(pVal);
     char *zRank;
@@ -938,6 +984,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
   pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
   pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
+  pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE;
 
   zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
   if( zSql ){

ext/fts5/fts5_expr.c

@@ -2477,7 +2477,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
   return pRet;
 }
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
   sqlite3_int64 nByte = 0;
   Fts5ExprTerm *p;
@@ -2844,14 +2844,14 @@ static void fts5ExprFold(
     sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics));
   }
 }
-#endif /* ifdef SQLITE_TEST */
+#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
 /*
 ** This is called during initialization to register the fts5_expr() scalar
 ** UDF with the SQLite handle passed as the only argument.
 */
 int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
   struct Fts5ExprFunc {
     const char *z;
     void (*x)(sqlite3_context*,int,sqlite3_value**);

ext/fts5/fts5_hash.c

@@ -475,7 +475,6 @@ static int fts5HashEntrySort(
     pList = fts5HashEntryMerge(pList, ap[i]);
   }
 
-  pHash->nEntry = 0;
   sqlite3_free(ap);
   *ppSorted = pList;
   return SQLITE_OK;
@@ -529,6 +528,28 @@ int sqlite3Fts5HashScanInit(
   return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
 }
 
+#ifdef SQLITE_DEBUG
+static int fts5HashCount(Fts5Hash *pHash){
+  int nEntry = 0;
+  int ii;
+  for(ii=0; ii<pHash->nSlot; ii++){
+    Fts5HashEntry *p = 0;
+    for(p=pHash->aSlot[ii]; p; p=p->pHashNext){
+      nEntry++;
+    }
+  }
+  return nEntry;
+}
+#endif
+
+/*
+** Return true if the hash table is empty, false otherwise.
+*/
+int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){
+  assert( pHash->nEntry==fts5HashCount(pHash) );
+  return pHash->nEntry==0;
+}
+
 void sqlite3Fts5HashScanNext(Fts5Hash *p){
   assert( !sqlite3Fts5HashScanEof(p) );
   p->pScan = p->pScan->pScanNext;

ext/fts5/fts5_main.c

@@ -1624,7 +1624,6 @@ static int fts5UpdateMethod(
   int eType0;                     /* value_type() of apVal[0] */
   int rc = SQLITE_OK;             /* Return code */
   int bUpdateOrDelete = 0;
-  
 
   /* A transaction must be open when this is called. */
   assert( pTab->ts.eState==1 || pTab->ts.eState==2 );
@@ -1654,7 +1653,14 @@ static int fts5UpdateMethod(
     if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
       && 0==sqlite3_stricmp("delete", z) 
     ){
-      rc = fts5SpecialDelete(pTab, apVal);
+      if( pConfig->bContentlessDelete ){
+        fts5SetVtabError(pTab, 
+            "'delete' may not be used with a contentless_delete=1 table"
+        );
+        rc = SQLITE_ERROR;
+      }else{
+        rc = fts5SpecialDelete(pTab, apVal);
+      }
     }else{
       rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
     }
@@ -1671,7 +1677,7 @@ static int fts5UpdateMethod(
     ** Cases 3 and 4 may violate the rowid constraint.
     */
     int eConflict = SQLITE_ABORT;
-    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
+    if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){
       eConflict = sqlite3_vtab_on_conflict(pConfig->db);
     }
 
@@ -1679,8 +1685,12 @@ static int fts5UpdateMethod(
     assert( nArg!=1 || eType0==SQLITE_INTEGER );
 
     /* Filter out attempts to run UPDATE or DELETE on contentless tables.
-    ** This is not suported.  */
-    if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){
+    ** This is not suported. Except - DELETE is supported if the CREATE
+    ** VIRTUAL TABLE statement contained "contentless_delete=1". */
+    if( eType0==SQLITE_INTEGER 
+     && pConfig->eContent==FTS5_CONTENT_NONE 
+     && pConfig->bContentlessDelete==0
+    ){
       pTab->p.base.zErrMsg = sqlite3_mprintf(
           "cannot %s contentless fts5 table: %s", 
           (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
@@ -1767,8 +1777,7 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
   fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
   pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
-  fts5TripCursors(pTab);
-  rc = sqlite3Fts5StorageSync(pTab->pStorage);
+  rc = sqlite3Fts5FlushToDisk(&pTab->p);
   pTab->p.pConfig->pzErrmsg = 0;
   return rc;
 }
@@ -2535,6 +2544,12 @@ static int fts5ColumnMethod(
       sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
     }
     pConfig->pzErrmsg = 0;
+  }else if( pConfig->bContentlessDelete && sqlite3_vtab_nochange(pCtx) ){
+    char *zErr = sqlite3_mprintf("cannot UPDATE a subset of "
+        "columns on fts5 contentless-delete table: %s", pConfig->zName
+    );
+    sqlite3_result_error(pCtx, zErr, -1);
+    sqlite3_free(zErr);
   }
   return rc;
 }

ext/fts5/fts5_storage.c

@@ -77,10 +77,10 @@ static int fts5StorageGetStmt(
       "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
       "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
       "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
-      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
+      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)",     /* REPLACE_DOCSIZE  */
       "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
 
-      "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */
+      "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?",    /* LOOKUP_DOCSIZE  */
 
       "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
       "SELECT %s FROM %s AS T",                         /* SCAN */
@@ -128,6 +128,19 @@ static int fts5StorageGetStmt(
         break;
       }
 
+      case FTS5_STMT_REPLACE_DOCSIZE: 
+        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName,
+          (pC->bContentlessDelete ? ",?" : "")
+        );
+        break;
+
+      case FTS5_STMT_LOOKUP_DOCSIZE: 
+        zSql = sqlite3_mprintf(azStmt[eStmt], 
+            (pC->bContentlessDelete ? ",origin" : ""),
+            pC->zDb, pC->zName
+        );
+        break;
+
       default:
         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
         break;
@@ -317,9 +330,11 @@ int sqlite3Fts5StorageOpen(
     }
 
     if( rc==SQLITE_OK && pConfig->bColumnsize ){
-      rc = sqlite3Fts5CreateTable(
-          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
-      );
+      const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
+      if( pConfig->bContentlessDelete ){
+        zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER";
+      }
+      rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
     }
     if( rc==SQLITE_OK ){
       rc = sqlite3Fts5CreateTable(
@@ -396,7 +411,7 @@ static int fts5StorageDeleteFromIndex(
 ){
   Fts5Config *pConfig = p->pConfig;
   sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
-  int rc;                         /* Return code */
+  int rc = SQLITE_OK;             /* Return code */
   int rc2;                        /* sqlite3_reset() return code */
   int iCol;
   Fts5InsertCtx ctx;
@@ -412,7 +427,6 @@ static int fts5StorageDeleteFromIndex(
 
   ctx.pStorage = p;
   ctx.iCol = -1;
-  rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
   for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
     if( pConfig->abUnindexed[iCol-1]==0 ){
       const char *zText;
@@ -449,6 +463,37 @@ static int fts5StorageDeleteFromIndex(
   return rc;
 }
 
+/*
+** This function is called to process a DELETE on a contentless_delete=1
+** table. It adds the tombstone required to delete the entry with rowid 
+** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs,
+** an SQLite error code.
+*/
+static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
+  i64 iOrigin = 0;
+  sqlite3_stmt *pLookup = 0;
+  int rc = SQLITE_OK;
+
+  assert( p->pConfig->bContentlessDelete );
+  assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
+
+  /* Look up the origin of the document in the %_docsize table. Store
+  ** this in stack variable iOrigin.  */
+  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pLookup, 1, iDel);
+    if( SQLITE_ROW==sqlite3_step(pLookup) ){
+      iOrigin = sqlite3_column_int64(pLookup, 1);
+    }
+    rc = sqlite3_reset(pLookup);
+  }
+
+  if( rc==SQLITE_OK && iOrigin!=0 ){
+    rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel);
+  }
+
+  return rc;
+}
 
 /*
 ** Insert a record into the %_docsize table. Specifically, do:
@@ -469,10 +514,17 @@ static int fts5StorageInsertDocsize(
     rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
     if( rc==SQLITE_OK ){
       sqlite3_bind_int64(pReplace, 1, iRowid);
-      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
-      sqlite3_step(pReplace);
-      rc = sqlite3_reset(pReplace);
-      sqlite3_bind_null(pReplace, 2);
+      if( p->pConfig->bContentlessDelete ){
+        i64 iOrigin = 0;
+        rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
+        sqlite3_bind_int64(pReplace, 3, iOrigin);
+      }
+      if( rc==SQLITE_OK ){
+        sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+        sqlite3_step(pReplace);
+        rc = sqlite3_reset(pReplace);
+        sqlite3_bind_null(pReplace, 2);
+      }
     }
   }
   return rc;
@@ -536,7 +588,15 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
 
   /* Delete the index records */
   if( rc==SQLITE_OK ){
-    rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
+    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
+  }
+
+  if( rc==SQLITE_OK ){
+    if( p->pConfig->bContentlessDelete ){
+      rc = fts5StorageContentlessDelete(p, iDel);
+    }else{
+      rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
+    }
   }
 
   /* Delete the %_docsize record */

ext/fts5/test/fts5contentless.test

@@ -0,0 +1,271 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+# Check that it is not possible to specify "contentless_delete=1" for 
+# anything other than a contentless table.
+#
+set res(0) {0 {}}
+set res(1) {1 {contentless_delete=1 requires a contentless table}}
+foreach {tn sql bError} {
+  1 "(a, b, contentless_delete=1)"              1
+  2 "(a, b, contentless_delete=1, content=abc)" 1
+  3 "(a, b, contentless_delete=1, content=)"    0
+  4 "(content=, contentless_delete=1, a)"       0
+  5 "(content='', contentless_delete=1, hello)" 0
+} {
+  execsql { BEGIN }
+  do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError)
+  execsql { ROLLBACK }
+}
+
+# Check that it is not possible to specify "contentless_delete=1" 
+# along with columnsize=1. 
+#
+set res(0) {0 {}}
+set res(1) {1 {contentless_delete=1 is incompatible with columnsize=0}}
+foreach {tn sql bError} {
+  2 "(a, b, content='', contentless_delete=1, columnsize=0)" 1 
+} {
+  execsql { BEGIN }
+  do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError)
+  execsql { ROLLBACK }
+}
+
+# Check that if contentless_delete=1 is specified, then the "origin" 
+# column is added to the %_docsize table.
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(c, content='');
+  CREATE VIRTUAL TABLE x2 USING fts5(c, content='', contentless_delete=1);
+}
+do_execsql_test 3.1 {
+  SELECT sql FROM sqlite_schema WHERE name IN ('x1_docsize', 'x2_docsize');
+} {
+  {CREATE TABLE 'x1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} 
+  {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER)}
+}
+
+do_execsql_test 3.2.1 {
+  SELECT hex(block) FROM x1_data WHERE id=10
+} {00000000000000}
+do_execsql_test 3.2.2 {
+  SELECT hex(block) FROM x2_data WHERE id=10
+} {00000000FF000001000000}
+
+do_execsql_test 3.3 {
+  INSERT INTO x2 VALUES('first text');
+  INSERT INTO x2 VALUES('second text');
+}
+do_execsql_test 3.4 {
+  SELECT id, origin FROM x2_docsize
+} {1 1 2 2}
+do_execsql_test 3.5 {
+  SELECT level, segment, loc1, loc2 FROM fts5_structure(
+    (SELECT block FROM x2_data WHERE id=10)
+  )
+} {
+  0 0   1 1
+  0 1   2 2
+}
+do_execsql_test 3.6 {
+  INSERT INTO x2(x2) VALUES('optimize');
+}
+do_execsql_test 3.7 {
+  SELECT level, segment, loc1, loc2 FROM fts5_structure(
+    (SELECT block FROM x2_data WHERE id=10)
+  )
+} {
+  1 0   1 2
+}
+
+do_execsql_test 3.8 {
+  DELETE FROM x2 WHERE rowid=2;
+}
+
+do_execsql_test 3.9 {
+  SELECT rowid FROM x2('text')
+} {1}
+
+#--------------------------------------------------------------------------
+reset_db
+proc document {n} {
+  set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+  set ret [list]
+  for {set ii 0} {$ii < $n} {incr ii} {
+    lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]]
+  }
+  set ret
+}
+
+set nRow 1000
+
+do_execsql_test 4.0 {
+  CREATE TABLE t1(x);
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 100);
+}
+do_test 4.1 {
+  for {set ii 0} {$ii < $nRow} {incr ii} {
+    set doc [document 6]
+    execsql {
+      INSERT INTO t1 VALUES($doc);
+      INSERT INTO ft VALUES($doc);
+    }
+  }
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+  set L2 [execsql {SELECT rowid FROM ft($v)}]
+  do_test 4.2.$v { set L1 } $L2
+}
+
+do_test 4.3 {
+  for {set ii 1} {$ii < $nRow} {incr ii 2} { 
+    execsql {
+       DELETE FROM ft WHERE rowid=$ii;
+       DELETE FROM t1 WHERE rowid=$ii;
+    }
+  }
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+  set L2 [execsql {SELECT rowid FROM ft($v)}]
+  do_test 4.4.$v { set L1 } $L2
+}
+
+do_execsql_test 4.5 {
+  INSERT INTO ft(ft) VALUES('optimize');
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+  set L2 [execsql {SELECT rowid FROM ft($v)}]
+  do_test 4.6.$v { set L1 } $L2
+}
+
+#execsql_pp { SELECT fts5_decode(id, block) FROM ft_data }
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(rowid, x) VALUES(1, 'one two three');
+  INSERT INTO ft(rowid, x) VALUES(2, 'one two four');
+  INSERT INTO ft(rowid, x) VALUES(3, 'one two five');
+  INSERT INTO ft(rowid, x) VALUES(4, 'one two seven');
+  INSERT INTO ft(rowid, x) VALUES(5, 'one two eight');
+}
+
+do_execsql_test 5.1 {
+  DELETE FROM ft WHERE rowid=2
+}
+
+do_execsql_test 5.2 {
+  SELECT rowid FROM ft
+} {1 3 4 5}
+
+do_catchsql_test 5.3 {
+  UPDATE ft SET x='four six' WHERE rowid=3
+} {0 {}}
+
+do_execsql_test 5.4 {
+  SELECT rowid FROM ft('one');
+} {1 4 5}
+
+do_execsql_test 5.5 {
+  REPLACE INTO ft(rowid, x) VALUES(3, 'four six');
+  SELECT rowid FROM ft('one');
+} {1 4 5}
+
+do_execsql_test 5.6 {
+  REPLACE INTO ft(rowid, x) VALUES(6, 'one two eleven');
+  SELECT rowid FROM ft('one');
+} {1 4 5 6}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(rowid, x) VALUES(1, 'one two three');
+  INSERT INTO ft(rowid, x) VALUES(2, 'one two four');
+}
+
+do_test 6.1 {
+  db eval { SELECT rowid FROM ft('one two') } {
+    if {$rowid==1} {
+      db eval { INSERT INTO ft(rowid, x) VALUES(3, 'one two four') }
+    }
+  }
+} {}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+}
+
+set lRowid [list -450 0 1 2 42]
+
+do_test 7.1 {
+  execsql BEGIN
+  foreach r $lRowid {
+    execsql { INSERT INTO ft(rowid, x) VALUES($r, 'one one one'); }
+  }
+  execsql COMMIT
+} {}
+
+do_test 7.2 {
+  execsql BEGIN
+  foreach r $lRowid {
+    execsql { REPLACE INTO ft(rowid, x) VALUES($r, 'two two two'); }
+  }
+  execsql COMMIT
+} {}
+
+do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {}
+do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft VALUES('hello world');
+  INSERT INTO ft VALUES('one two three');
+}
+
+do_catchsql_test 8.1 {
+  INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world');
+} {1 {'delete' may not be used with a contentless_delete=1 table}}
+
+do_execsql_test 8.2 {
+  BEGIN;
+    INSERT INTO ft(rowid, x) VALUES(3, 'four four four');
+    DELETE FROM ft WHERE rowid=3;
+  COMMIT;
+  SELECT rowid FROM ft('four');
+} {}
+
+finish_test
+

ext/fts5/test/fts5contentless2.test

@@ -0,0 +1,208 @@
+# 2023 July 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc vocab {} {
+  list aaa bbb ccc ddd eee fff ggg hhh iii jjj kkk lll mmm nnn ooo ppp
+}
+
+proc document {nToken} {
+  set doc [list]
+  set vocab [vocab]
+  for {set ii 0} {$ii < $nToken} {incr ii} {
+    lappend doc [lindex $vocab [expr int(rand()*[llength $vocab])]]
+  }
+  set doc
+}
+db func document document
+
+proc contains {doc token} {
+  expr {[lsearch $doc $token]>=0}
+}
+db func contains contains
+
+proc do_compare_tables_test {tn} {
+  uplevel [list do_test $tn {
+    foreach v [vocab] {
+      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $v) }]
+      set l2 [execsql { SELECT rowid FROM t2($v) }]
+      if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" }
+
+      set w "[string range $v 0 1]*"
+      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }]
+      set l2 [execsql { SELECT rowid FROM t2($w) }]
+      if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" }
+
+      set w "[string range $v 0 0]*"
+      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }]
+      set l2 [execsql { SELECT rowid FROM t2($w) }]
+      if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" }
+
+      set l1 [execsql { 
+        SELECT rowid FROM t1 WHERE contains(doc, $v) ORDER BY rowid DESC 
+      }]
+      set l2 [execsql { SELECT rowid FROM t2($v) ORDER BY rowid DESC }]
+      if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" }
+    }
+    set {} {}
+  } {}]
+}
+
+proc lshuffle {in} {
+  set L [list]
+  set ret [list]
+  foreach elem $in { lappend L [list [expr rand()] $elem] }
+  foreach pair [lsort -index 0 $L] { lappend ret [lindex $pair 1] }
+  set ret
+}
+
+expr srand(0)
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      doc, prefix=2, content=, contentless_delete=1
+  );
+
+  CREATE TABLE t1(doc);
+  CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN
+    DELETE FROM t2 WHERE rowid = old.rowid;
+  END;
+}
+
+set SMALLEST64 -9223372036854775808
+set LARGEST64   9223372036854775807
+
+foreach {tn r1 r2} {
+  1   0               50
+  2   $SMALLEST64     $SMALLEST64+50
+  3   $LARGEST64-50   $LARGEST64
+  4   -50             -1
+} {
+  set r1 [expr $r1]
+  set r2 [expr $r2]
+
+  do_test 1.1.$tn {
+    execsql BEGIN
+    for {set ii $r1} {$ii <= $r2} {incr ii} {
+      execsql { INSERT INTO t1(rowid, doc) VALUES ($ii, document(8)); }
+    }
+    execsql COMMIT
+  } {}
+}
+do_test 1.2 {
+  db eval { SELECT rowid, doc FROM t1 } {
+    execsql { INSERT INTO t2(rowid, doc) VALUES($rowid, $doc) }
+  }
+} {}
+
+foreach {tn rowid} {
+  1  $SMALLEST64
+  2  0
+  3  -5
+  4  -30
+  5  $LARGEST64
+  6  $LARGEST64-1
+} {
+  set rowid [expr $rowid]
+  do_execsql_test 1.3.$tn.1 {
+    DELETE FROM t1 WHERE rowid=$rowid
+  }
+  do_compare_tables_test 1.3.$tn.2
+}
+
+set iTest 1
+foreach r [lshuffle [execsql {SELECT rowid FROM t1}]] {
+  if {($iTest % 50)==0} {
+    execsql { INSERT INTO t2(t2) VALUES('optimize') }
+  }
+  if {($iTest % 5)==0} {
+    execsql { INSERT INTO t2(t2, rank) VALUES('merge', 5) }
+  }
+  do_execsql_test 1.4.$iTest.1($r) {
+    DELETE FROM t1 WHERE rowid=$r
+  }
+  do_compare_tables_test 1.4.$iTest.2
+  incr iTest
+}
+
+do_execsql_test 1.5 {
+  SELECT * FROM t1
+} {}
+
+#-------------------------------------------------------------------------
+reset_db
+db func document document
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s;
+}
+
+do_execsql_test 2.1 {
+  BEGIN;
+    DELETE FROM t2 WHERE rowid=32;
+    DELETE FROM t2 WHERE rowid=64;
+    DELETE FROM t2 WHERE rowid=96;
+    DELETE FROM t2 WHERE rowid=128;
+    DELETE FROM t2 WHERE rowid=160;
+    DELETE FROM t2 WHERE rowid=192;
+  COMMIT;
+}
+
+do_execsql_test 2.2 {
+  SELECT * FROM t2('128');
+} {}
+
+#-------------------------------------------------------------------------
+
+foreach {tn step} {
+  1     3 
+  2     7
+  3     15
+} {
+  set step [expr $step]
+
+  reset_db
+  db func document document
+  do_execsql_test 3.$tn.0 {
+    CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1);
+    INSERT INTO t2(t2, rank) VALUES('pgsz', 100);
+    WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+    )
+    INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s;
+  }
+  do_execsql_test 3.$tn.1 {
+    DELETE FROM t2 WHERE (rowid % $step)==0
+  }
+  do_execsql_test 3.$tn.2 {
+    SELECT * FROM t2( $step * 5 )
+  } {}
+}
+
+
+
+finish_test
+

ext/fts5/test/fts5contentless3.test

@@ -0,0 +1,196 @@
+# 2023 July 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO ft VALUES('one one one');
+    INSERT INTO ft VALUES('two two two');
+    INSERT INTO ft VALUES('three three three');
+    INSERT INTO ft VALUES('four four four');
+    INSERT INTO ft VALUES('five five five');
+    INSERT INTO ft VALUES('six six six');
+    INSERT INTO ft VALUES('seven seven seven');
+    INSERT INTO ft VALUES('eight eight eight');
+    INSERT INTO ft VALUES('nine nine nine');
+  COMMIT;
+
+  DELETE FROM ft WHERE rowid=3;
+}
+
+proc myhex {hex} { binary decode hex $hex }
+db func myhex myhex
+
+do_execsql_test 1.1 {
+  UPDATE ft_data SET block = 
+    myhex('04000000 00000001'                   ||
+          '01020304 01020304 01020304 01020304' ||
+          '01020304 01020304 01020304 01020304'
+    )
+  WHERE id = (SELECT max(id) FROM ft_data);
+}
+
+do_execsql_test 1.2 {
+  DELETE FROM ft WHERE rowid=1
+}
+
+do_execsql_test 1.3 {
+  SELECT rowid FROM ft('two');
+} {2}
+
+do_execsql_test 1.3 {
+  UPDATE ft_data SET block = 
+    myhex('08000000 00000001'                                               ||
+      '0000000001020304 0000000001020304 0000000001020304 0000000001020304' ||
+      '0000000001020304 0000000001020304 0000000001020304 0000000001020304'
+    )
+  WHERE id = (SELECT max(id) FROM ft_data);
+}
+
+do_execsql_test 1.4 {
+  SELECT rowid FROM ft('two');
+} {2}
+
+do_execsql_test 1.5 {
+  DELETE FROM ft WHERE rowid=4
+}
+
+do_execsql_test 1.6 {
+  UPDATE ft_data SET block = myhex('04000000 00000000')
+    WHERE id = (SELECT max(id) FROM ft_data);
+}
+do_execsql_test 1.7 {
+  SELECT rowid FROM ft('two');
+} {2}
+
+do_execsql_test 1.8 {
+  UPDATE ft_data SET block = myhex('04000000 00000000')
+    WHERE id = (SELECT max(id) FROM ft_data);
+}
+do_execsql_test 1.9 {
+  DELETE FROM ft WHERE rowid=8
+} {}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
+  INSERT INTO ft VALUES('one one one');
+  INSERT INTO ft VALUES('two two two');
+  INSERT INTO ft VALUES('three three three');
+  INSERT INTO ft VALUES('four four four');
+  INSERT INTO ft VALUES('five five five');
+  INSERT INTO ft VALUES('six six six');
+  INSERT INTO ft VALUES('seven seven seven');
+  INSERT INTO ft VALUES('eight eight eight');
+  INSERT INTO ft VALUES('nine nine nine');
+}
+
+do_execsql_test 2.1 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+do_execsql_test 2.2 {
+  SELECT count(*) FROM ft_data
+} {3}
+do_execsql_test 2.3 {
+  DELETE FROM ft WHERE rowid=5
+}
+do_execsql_test 2.4 {
+  SELECT count(*) FROM ft_data
+} {4}
+
+# Check that an 'optimize' works (rewrites the index) if there is a single
+# segment with one or more tombstone hash pages.
+do_execsql_test 2.5 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+do_execsql_test 2.6 {
+  SELECT count(*) FROM ft_data
+} {3}
+
+# Check that an 'optimize' is a no-op if there is a single segment
+# and no tombstone hash pages.
+do_execsql_test 2.7 {
+  INSERT INTO ft(ft) VALUES('optimize');
+  SELECT rowid FROM ft_data;
+} [db eval {SELECT rowid FROM ft_data}]
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO ft(rowid, x) SELECT i, i||' '||i||' '||i||' '||i FROM s;
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+do_execsql_test 3.1 {
+  SELECT count(*) FROM ft_data
+} {200}
+
+do_execsql_test 3.2 {
+  DELETE FROM ft WHERE (rowid % 50)==0;
+  SELECT count(*) FROM ft_data;
+} {203}
+
+do_execsql_test 3.3 {
+  INSERT INTO ft(ft, rank) VALUES('merge', 500);
+  SELECT rowid FROM ft_data;
+} [db eval {SELECT rowid FROM ft_data}]
+
+do_execsql_test 3.4 {
+  INSERT INTO ft(ft, rank) VALUES('merge', -1000);
+  SELECT count(*) FROM ft_data;
+} {197}
+
+do_execsql_test 3.5 {
+  DELETE FROM ft WHERE (rowid % 50)==1;
+  SELECT count(*) FROM ft_data;
+} {200}
+
+do_execsql_test 3.6 {
+  SELECT level, segment, npgtombstone FROM fts5_structure(
+    (SELECT block FROM ft_data WHERE id=10)
+  )
+} {1 0 3}
+
+do_test 3.6 {
+  while 1 {
+    set nChange [db total_changes]
+    execsql { INSERT INTO ft(ft, rank) VALUES('merge', -5) }
+    if {([db total_changes] - $nChange)<2} break
+  }
+} {}
+
+do_execsql_test 3.7 {
+  SELECT level, segment, npgtombstone FROM fts5_structure(
+    (SELECT block FROM ft_data WHERE id=10)
+  )
+} {2 0 0}
+
+
+finish_test
+

ext/fts5/test/fts5contentless4.test

@@ -0,0 +1,248 @@
+# 2023 July 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc document {n} {
+  set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+  set ret [list]
+  for {set ii 0} {$ii < $n} {incr ii} {
+    lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]]
+  }
+  set ret
+}
+db func document document
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 240);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO ft SELECT document(12) FROM s;
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+do_execsql_test 1.2 {
+  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {0 0 1000 0}
+
+do_execsql_test 1.3 {
+  DELETE FROM ft WHERE rowid < 50
+}
+
+do_execsql_test 1.4 {
+  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {0 0 1000 49}
+
+do_execsql_test 1.5 {
+  DELETE FROM ft WHERE rowid < 1000
+}
+
+do_execsql_test 1.6 {
+  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {1 0 1 0}
+
+#--------------------------------------------------------------------------
+reset_db
+db func document document
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+}
+
+do_test 2.1 {
+  for {set ii 0} {$ii < 5000} {incr ii} {
+    execsql { INSERT INTO ft VALUES( document(12) ) }
+  }
+} {}
+
+do_execsql_test 2.2 {
+  SELECT sum(nentry) - sum(nentrytombstone) FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {5000}
+
+for {set ii 5000} {$ii >= 0} {incr ii -100} {
+  do_execsql_test 2.3.$ii {
+    DELETE FROM ft WHERE rowid > $ii
+  }
+  do_execsql_test 2.3.$ii.2 {
+    SELECT 
+      CAST((total(nentry) - total(nentrytombstone)) AS integer) 
+    FROM 
+      fts5_structure( (SELECT block FROM ft_data WHERE id=10) )
+  } $ii
+}
+
+execsql_pp {
+  SELECT * FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+}
+
+do_test 2.4 {
+  for {set ii 0} {$ii < 5000} {incr ii} {
+    execsql { INSERT INTO ft VALUES( document(12) ) }
+  }
+} {}
+
+for {set ii 1} {$ii <= 5000} {incr ii 10} {
+  do_execsql_test 2.3.$ii {
+    DELETE FROM ft WHERE rowid = $ii;
+    INSERT INTO ft VALUES( document(12) );
+    INSERT INTO ft(ft, rank) VALUES('merge', -10);
+  }
+
+  do_execsql_test 2.3.$ii.2 {
+    SELECT 
+      CAST((total(nentry) - total(nentrytombstone)) AS integer) 
+    FROM 
+      fts5_structure( (SELECT block FROM ft_data WHERE id=10) )
+  } 5000
+}
+
+#-------------------------------------------------------------------------
+reset_db
+db func document document
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO ft SELECT document(12) FROM s;
+}
+
+do_catchsql_test 3.1 {
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 'text');
+} {1 {SQL logic error}}
+do_catchsql_test 3.2 {
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 50);
+} {0 {}}
+do_execsql_test 3.3 {
+  SELECT * FROM ft_config WHERE k='deletemerge'
+} {deletemerge 50}
+do_catchsql_test 3.4 {
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 101);
+} {0 {}}
+do_execsql_test 3.5 {
+  SELECT * FROM ft_config WHERE k='deletemerge'
+} {deletemerge 101}
+
+do_execsql_test 3.6 {
+  DELETE FROM ft WHERE rowid<95
+}
+
+do_execsql_test 3.7 {
+  SELECT nentrytombstone, nentry FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {94 100}
+
+do_execsql_test 3.8 {
+  DELETE FROM ft WHERE rowid=95
+}
+
+do_execsql_test 3.9 {
+  SELECT nentrytombstone, nentry FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {95 100}
+
+do_execsql_test 3.10 {
+  DELETE FROM ft;
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO ft SELECT document(12) FROM s;
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 50);
+}
+
+do_execsql_test 3.11 {
+  DELETE FROM ft WHERE rowid<95
+}
+
+do_execsql_test 3.12 {
+  SELECT nentrytombstone, nentry FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {0 6}
+
+#-------------------------------------------------------------------------
+reset_db
+db func document document
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO x1(x1, rank) VALUES('usermerge', 16);
+  INSERT INTO x1(x1, rank) VALUES('deletemerge', 40);
+  INSERT INTO x1 VALUES('one');
+  INSERT INTO x1 VALUES('two');
+  INSERT INTO x1 VALUES('three');
+  INSERT INTO x1 VALUES('four');
+  INSERT INTO x1 VALUES('five');
+  INSERT INTO x1 VALUES('six');
+  INSERT INTO x1 VALUES('seven');
+  INSERT INTO x1 VALUES('eight');
+  INSERT INTO x1 VALUES('nine');
+  INSERT INTO x1 VALUES('ten');
+}
+
+do_execsql_test 4.1 {
+  SELECT level, segment FROM fts5_structure((
+    SELECT block FROM x1_data WHERE id=10
+  ))
+} {
+  0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9
+}
+
+for {set ii 1} {$ii < 4} {incr ii} {
+  do_execsql_test 4.2.$ii {
+    DELETE FROM x1 WHERE rowid = $ii;
+    INSERT INTO x1(x1, rank) VALUES('merge', 5);
+    SELECT level, segment FROM fts5_structure((
+          SELECT block FROM x1_data WHERE id=10
+    ))
+  } {
+    0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9
+  }
+}
+
+do_execsql_test 4.3 {
+  DELETE FROM x1 WHERE rowid = $ii;
+  INSERT INTO x1(x1, rank) VALUES('merge', 5);
+  SELECT level, segment, nentry FROM fts5_structure((
+        SELECT block FROM x1_data WHERE id=10
+  ))
+} {
+  1 0   6
+}
+
+finish_test
+

ext/fts5/test/fts5contentless5.test

@@ -0,0 +1,59 @@
+# 2023 August 7
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless5
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, content='', contentless_delete=1);
+  INSERT INTO t1 VALUES('A', 'B', 'C');
+  INSERT INTO t1 VALUES('D', 'E', 'F');
+  INSERT INTO t1 VALUES('G', 'H', 'I');
+}
+
+do_execsql_test 1.01 {
+  CREATE TABLE t2(x, y);
+  INSERT INTO t2 VALUES('x', 'y');
+}
+
+# explain_i "UPDATE t1 SET a='a' WHERE t1.rowid=1"    
+breakpoint
+explain_i "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1 AND b IS NULL"     
+
+#breakpoint
+#explain_i "UPDATE t1 SET a='a' WHERE b IS NULL AND rowid=?"
+
+foreach {tn up err} {
+  1   "UPDATE t1 SET a='a', b='b', c='c' WHERE rowid=1"                  0
+  2   "UPDATE t1 SET a='a', b='b'        WHERE rowid=1"                  1
+  3   "UPDATE t1 SET        b='b', c='c' WHERE rowid=1"                  1
+  4   "UPDATE t1 SET a='a',        c='c' WHERE rowid=1"                  1
+  5   "UPDATE t1 SET a='a',        c='c' WHERE t1.rowid=1 AND b IS NULL" 1
+  6   "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1"                     1
+  7   "UPDATE t1 SET a='a', b='b', c='c' FROM t2 WHERE t1.rowid=1"       0
+} {
+
+  set res(0) {0 {}}
+  set res(1) {1 {cannot UPDATE a subset of columns on fts5 contentless-delete table: t1}}
+  do_catchsql_test 1.$tn $up $res($err)
+}
+
+finish_test
+

ext/fts5/test/fts5faultF.test

@@ -0,0 +1,111 @@
+# 2023 July 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors. Particularly those that may occur
+# when using contentless_delete=1 databases.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultF
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+faultsim_save_and_close
+do_faultsim_test 1 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    CREATE VIRTUAL TABLE t1 USING fts5(x, y, content=, contentless_delete=1) 
+  }
+} -test {
+  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} 
+}
+
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d');
+  COMMIT;
+  DELETE FROM t1 WHERE rowid IN (2, 4);
+}
+
+do_faultsim_test 2 -prep {
+  sqlite3 db test.db
+  execsql { SELECT rowid FROM t1 }
+} -body {
+  execsql { 
+    SELECT rowid FROM t1('b c');
+  }
+} -test {
+  faultsim_test_result {0 {1 3}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d');
+  COMMIT;
+}
+
+faultsim_save_and_close
+do_faultsim_test 3 -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT rowid FROM t1 }
+} -body {
+  execsql { 
+    INSERT INTO t1(rowid, doc) VALUES(5, 'a b c d');
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO t1(rowid, doc) SELECT i, 'a b c d' FROM s;
+}
+
+do_execsql_test 4.1 { DELETE FROM t1 WHERE rowid <= 25 }
+
+faultsim_save_and_close
+do_faultsim_test 4 -faults oom-t* -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT rowid FROM t1 }
+} -body {
+  execsql { 
+    DELETE FROM t1 WHERE rowid < 100
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+
+finish_test
+

ext/misc/bgckpt.c

@@ -0,0 +1,244 @@
+/*
+** 2017-10-11
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+
+#if !defined(SQLITE_TEST) || defined(SQLITE_OS_UNIX)
+
+#include "sqlite3.h"
+#include <string.h>
+#include <pthread.h>
+
+/*
+** API declarations.
+*/
+typedef struct Checkpointer Checkpointer;
+int sqlite3_bgckpt_create(const char *zFilename, Checkpointer **pp);
+int sqlite3_bgckpt_checkpoint(Checkpointer *p, int bBlock);
+void sqlite3_bgckpt_destroy(Checkpointer *p);
+
+
+struct Checkpointer {
+  sqlite3 *db;                    /* Database handle */
+
+  pthread_t thread;               /* Background thread */
+  pthread_mutex_t mutex;
+  pthread_cond_t cond;
+
+  int rc;                         /* Error from "PRAGMA wal_checkpoint" */
+  int bCkpt;                      /* True if checkpoint requested */
+  int bExit;                      /* True if exit requested */
+};
+
+static void *bgckptThreadMain(void *pCtx){
+  int rc = SQLITE_OK;
+  Checkpointer *p = (Checkpointer*)pCtx;
+
+  while( rc==SQLITE_OK ){
+    int bExit;
+
+    pthread_mutex_lock(&p->mutex);
+    if( p->bCkpt==0 && p->bExit==0 ){
+      pthread_cond_wait(&p->cond, &p->mutex);
+    }
+    p->bCkpt = 0;
+    bExit = p->bExit;
+    pthread_mutex_unlock(&p->mutex);
+
+    if( bExit ) break;
+    rc = sqlite3_exec(p->db, "PRAGMA wal_checkpoint", 0, 0, 0);
+    if( rc==SQLITE_BUSY ){
+      rc = SQLITE_OK;
+    }
+  }
+
+  pthread_mutex_lock(&p->mutex);
+  p->rc = rc;
+  pthread_mutex_unlock(&p->mutex);
+  return 0;
+}
+
+void sqlite3_bgckpt_destroy(Checkpointer *p){
+  if( p ){
+    void *ret = 0;
+
+    /* Signal the background thread to exit */
+    pthread_mutex_lock(&p->mutex);
+    p->bExit = 1;
+    pthread_cond_broadcast(&p->cond);
+    pthread_mutex_unlock(&p->mutex);
+
+    pthread_join(p->thread, &ret);
+    sqlite3_close(p->db);
+    sqlite3_free(p);
+  }
+}
+
+
+int sqlite3_bgckpt_create(const char *zFilename, Checkpointer **pp){
+  Checkpointer *pNew = 0;
+  int rc;
+
+  pNew = (Checkpointer*)sqlite3_malloc(sizeof(Checkpointer));
+  if( pNew==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    memset(pNew, 0, sizeof(Checkpointer));
+    rc = sqlite3_open(zFilename, &pNew->db);
+  }
+
+  if( rc==SQLITE_OK ){
+    pthread_mutex_init(&pNew->mutex, 0);
+    pthread_cond_init(&pNew->cond, 0);
+    pthread_create(&pNew->thread, 0, bgckptThreadMain, (void*)pNew);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_bgckpt_destroy(pNew);
+    pNew = 0;
+  }
+  *pp = pNew;
+  return rc;
+}
+
+int sqlite3_bgckpt_checkpoint(Checkpointer *p, int bBlock){
+  int rc;
+  pthread_mutex_lock(&p->mutex);
+  rc = p->rc;
+  if( rc==SQLITE_OK ){
+    p->bCkpt = 1;
+    pthread_cond_broadcast(&p->cond);
+  }
+  pthread_mutex_unlock(&p->mutex);
+  return rc;
+}
+
+#ifdef SQLITE_TEST
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+
+const char *sqlite3ErrName(int rc);
+
+static void SQLITE_TCLAPI bgckpt_del(void * clientData){
+  Checkpointer *pCkpt = (Checkpointer*)clientData;
+  sqlite3_bgckpt_destroy(pCkpt);
+}
+
+/*
+** Tclcmd: $ckpt SUBCMD ...
+*/
+static int SQLITE_TCLAPI bgckpt_obj_cmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  Checkpointer *pCkpt = (Checkpointer*)clientData;
+  const char *aCmd[] = { "checkpoint", "destroy", 0 };
+  int iCmd;
+
+  if( objc<2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SUBCMD ...");
+    return TCL_ERROR;
+  }
+
+  if( Tcl_GetIndexFromObj(interp, objv[1], aCmd, "sub-command", 0, &iCmd) ){
+    return TCL_ERROR;
+  }
+
+  switch( iCmd ){
+    case 0: {
+      int rc;
+      int bBlock = 0;
+
+      if( objc>3 ){
+        Tcl_WrongNumArgs(interp, 2, objv, "?BLOCKING?");
+        return TCL_ERROR;
+      }
+      if( objc==3 && Tcl_GetBooleanFromObj(interp, objv[2], &bBlock) ){
+        return TCL_ERROR;
+      }
+
+      rc = sqlite3_bgckpt_checkpoint(pCkpt, bBlock);
+      if( rc!=SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+        return TCL_ERROR;
+      }
+      break;
+    }
+
+    case 1: {
+      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+      break;
+    }
+  }
+
+  return TCL_OK;
+}
+
+/*
+** Tclcmd: bgckpt CMDNAME FILENAME
+*/
+static int SQLITE_TCLAPI bgckpt_cmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  const char *zCmd;
+  const char *zFilename;
+  int rc;
+  Checkpointer *pCkpt;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "CMDNAME FILENAME");
+    return TCL_ERROR;
+  }
+  zCmd = Tcl_GetString(objv[1]);
+  zFilename = Tcl_GetString(objv[2]);
+
+  rc = sqlite3_bgckpt_create(zFilename, &pCkpt);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    return TCL_ERROR;
+  }
+
+  Tcl_CreateObjCommand(interp, zCmd, bgckpt_obj_cmd, (void*)pCkpt, bgckpt_del);
+  Tcl_SetObjResult(interp, objv[1]);
+  return TCL_OK;
+}
+
+int Bgckpt_Init(Tcl_Interp *interp){
+  Tcl_CreateObjCommand(interp, "bgckpt", bgckpt_cmd, 0, 0);
+  return TCL_OK;
+}
+#endif   /* SQLITE_TEST */
+
+#else
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+int Bgckpt_Init(Tcl_Interp *interp){ return TCL_OK; }
+#endif
+

ext/session/sqlite3session.c

@@ -891,6 +891,7 @@ static int sessionPreupdateEqual(
         rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal);
       }
       assert( rc==SQLITE_OK );
+      (void)rc;                   /* Suppress warning about unused variable */
       if( sqlite3_value_type(pVal)!=eType ) return 0;
 
       /* A SessionChange object never has a NULL value in a PK column */

ext/wasm/EXPORTED_FUNCTIONS.fiddle

@@ -1,10 +1,7 @@
+_fiddle_exec
+_fiddle_interrupt
+_fiddle_experiment
+_fiddle_the_db
 _fiddle_db_arg
 _fiddle_db_filename
-_fiddle_exec
-_fiddle_experiment
-_fiddle_interrupt
-_fiddle_main
 _fiddle_reset_db
-_fiddle_db_handle
-_fiddle_db_vfs
-_fiddle_export_db

ext/wasm/GNUmakefile

@@ -18,8 +18,6 @@
 #  quick, q = do just a minimal build (sqlite3.js/wasm, tester1) for
 #      faster development-mode turnaround.
 #
-#  qo2, qoz = a combination of quick+o2/oz.
-#
 #  dist = create end user deliverables. Add dist.build=oX to build
 #      with a specific optimization level, where oX is one of the
 #      above-listed o? or qo? target names.
@@ -46,11 +44,12 @@
 #    $(eval), or at least centralize the setup of the numerous vars
 #    related to each build variant $(JS_BUILD_MODES).
 #
+default: all
+#default: quick
 SHELL := $(shell which bash 2>/dev/null)
 MAKEFILE := $(lastword $(MAKEFILE_LIST))
 CLEAN_FILES :=
 DISTCLEAN_FILES := ./--dummy--
-default: all
 release: oz
 # JS_BUILD_MODES exists solely to reduce repetition in documentation
 # below.
@@ -182,7 +181,7 @@ SQLITE_OPT = \
   -DSQLITE_OMIT_SHARED_CACHE \
   -DSQLITE_OMIT_WAL \
   -DSQLITE_THREADSAFE=0 \
-  -DSQLITE_TEMP_STORE=3 \
+  -DSQLITE_TEMP_STORE=2 \
   -DSQLITE_OS_KV_OPTIONAL=1 \
   '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \
   -DSQLITE_USE_URI=1 \
@@ -375,6 +374,7 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js
 sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js
 sqlite3-api.jses += $(dir.api)/sqlite3-v-helper.js
 sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js
+sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js
 sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js
 
 # SOAP.js is an external API file which is part of our distribution
@@ -451,6 +451,13 @@ emcc.exportedRuntimeMethods := \
 emcc.jsflags += $(emcc.exportedRuntimeMethods)
 emcc.jsflags += -sUSE_CLOSURE_COMPILER=0
 emcc.jsflags += -sIMPORTED_MEMORY
+#emcc.jsflags += -sASYNCIFY=2
+# ^^^ ASYNCIFY=2 is for experimental JSPI support
+# (https://v8.dev/blog/jspi), but enabling it causes the lib-level
+# init code to throw inexplicable complaints about C-level function
+# signatures not matching what we expect them to be. JSPI requires, as of
+# this writing, requires an experimental Chrome flag:
+# chrome://flags/#enable-experimental-webassembly-stack-switching
 emcc.jsflags += -sSTRICT_JS=0
 # STRICT_JS disabled due to:
 #   https://github.com/emscripten-core/emscripten/issues/18610
@@ -677,10 +684,15 @@ sqlite3-wasmfs.cfiles := $(sqlite3-wasm.cfiles)
 #
 # Upstream RFE:
 # https://github.com/emscripten-core/emscripten/issues/18237
+#
+# Maintenance reminder: Mac sed works differently than GNU sed, so
+# don't use sed for this.
 define SQLITE3.xJS.ESM-EXPORT-DEFAULT
 if [ x1 = x$(1) ]; then \
 		echo "Fragile workaround for emscripten/issues/18237. See SQLITE3.xJS.RECIPE."; \
-		sed -i -e '0,/^export default/{/^export default/d;}' $@ || exit $$?; \
+		{\
+			awk '/^export default/ && !f{f=1; next} 1' $@ > $@.tmp && mv $@.tmp $@; \
+		} || exit $$?; \
 		if [ x != x$(2) ]; then \
 			if ! grep -q '^export default' $@; then \
 				echo "Cannot find export default." 1>&2; \
@@ -754,7 +766,7 @@ $(5): $(4) $$(MAKEFILE) $$(sqlite3-wasm.cfiles) $$(EXPORTED_FUNCTIONS.api) $$(pr
 	esac; \
 	ls -la $$$$dotwasm $$@
 all: $(5)
-quick: $(5)
+#quick: $(5)
 CLEAN_FILES += $(4) $(5)
 endef
 # ^^^ /SETUP_LIB_BUILD_MODE
@@ -858,7 +870,7 @@ clean-batch:
 # a regular basis with different -Ox flags and rebuilding the batch
 # pieces each time is an unnecessary time sink.
 batch: batch-runner.list
-all: batch
+#all: batch
 # end batch-runner.js
 ########################################################################
 # Wasmified speedtest1 is our primary benchmarking tool.
@@ -959,6 +971,7 @@ tester1: tester1.js tester1.mjs tester1.html tester1-esm.html
 # Note that we do not include $(sqlite3-bundler-friendly.mjs) in this
 # because bundlers are client-specific.
 all quick: tester1
+quick: $(sqlite3.js)
 
 ########################################################################
 # Convenience rules to rebuild with various -Ox levels. Much
@@ -978,8 +991,6 @@ o1: clean
 	$(MAKE) -e "emcc_opt=-O1 $(o-xtra)"
 o2: clean
 	$(MAKE) -j2 -e "emcc_opt=-O2 $(o-xtra)"
-qo2: clean
-	$(MAKE) -j2 -e "emcc_opt=-O2 $(o-xtra)" quick
 o3: clean
 	$(MAKE) -e "emcc_opt=-O3 $(o-xtra)"
 os: clean
@@ -987,8 +998,6 @@ os: clean
 	$(MAKE) -e "emcc_opt=-Os $(o-xtra)"
 oz: clean
 	$(MAKE) -j2 -e "emcc_opt=-Oz $(o-xtra)"
-qoz: clean
-	$(MAKE) -j2 -e "emcc_opt=-Oz $(o-xtra)" quick
 
 ########################################################################
 # Sub-makes...

ext/wasm/README.md

@@ -82,7 +82,7 @@ features in the apps which use them.
 
 # Testing on a remote machine that is accessed via SSH
 
-*NB: The following are developer notes, last validated on 2022-08-18*
+*NB: The following are developer notes, last validated on 2023-07-19*
 
   *  Remote: Install git, emsdk, and althttpd
      *  Use a [version of althttpd][althttpd] from
@@ -93,13 +93,14 @@ features in the apps which use them.
   *  Local:  `ssh -L 8180:localhost:8080 remote`
   *  Local:  Point your web-browser at http://localhost:8180/index.html
 
-In order to enable [SharedArrayBuffers](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer),
-the web-browser requires that the two extra Cross-Origin lines be present
-in HTTP reply headers and that the request must come from "localhost".
-Since the web-server is on a different machine from
-the web-broser, the localhost requirement means that the connection must be tunneled
-using SSH.
+In order to enable [SharedArrayBuffer][], the web-browser requires
+that the two extra Cross-Origin lines be present in HTTP reply headers
+and that the request must come from "localhost" (_or_ over an SSL
+connection).  Since the web-server is on a different machine from the
+web-broser, the localhost requirement means that the connection must
+be tunneled using SSH.
 
 
 [emscripten]: https://emscripten.org
 [althttpd]: https://sqlite.org/althttpd
+[SharedArrayBuffer]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer

ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api

@@ -1,8 +1,3 @@
-_malloc
-_free
-_realloc
-_sqlite3_aggregate_context
-_sqlite3_auto_extension
 _sqlite3_bind_blob
 _sqlite3_bind_double
 _sqlite3_bind_int
@@ -10,16 +5,11 @@ _sqlite3_bind_int64
 _sqlite3_bind_null
 _sqlite3_bind_parameter_count
 _sqlite3_bind_parameter_index
-_sqlite3_bind_pointer
 _sqlite3_bind_text
-_sqlite3_busy_handler
-_sqlite3_busy_timeout
-_sqlite3_cancel_auto_extension
 _sqlite3_changes
 _sqlite3_changes64
 _sqlite3_clear_bindings
 _sqlite3_close_v2
-_sqlite3_collation_needed
 _sqlite3_column_blob
 _sqlite3_column_bytes
 _sqlite3_column_count
@@ -30,28 +20,12 @@ _sqlite3_column_int64
 _sqlite3_column_name
 _sqlite3_column_text
 _sqlite3_column_type
-_sqlite3_column_value
-_sqlite3_commit_hook
 _sqlite3_compileoption_get
 _sqlite3_compileoption_used
-_sqlite3_complete
-_sqlite3_context_db_handle
-_sqlite3_create_collation
-_sqlite3_create_collation_v2
-_sqlite3_create_function
 _sqlite3_create_function_v2
-_sqlite3_create_module
-_sqlite3_create_module_v2
-_sqlite3_create_window_function
 _sqlite3_data_count
 _sqlite3_db_filename
-_sqlite3_db_handle
 _sqlite3_db_name
-_sqlite3_db_status
-_sqlite3_declare_vtab
-_sqlite3_deserialize
-_sqlite3_drop_modules
-_sqlite3_errcode
 _sqlite3_errmsg
 _sqlite3_error_offset
 _sqlite3_errstr
@@ -59,38 +33,16 @@ _sqlite3_exec
 _sqlite3_expanded_sql
 _sqlite3_extended_errcode
 _sqlite3_extended_result_codes
-_sqlite3_file_control
 _sqlite3_finalize
-_sqlite3_free
-_sqlite3_get_auxdata
 _sqlite3_initialize
-_sqlite3_keyword_count
-_sqlite3_keyword_name
-_sqlite3_keyword_check
-_sqlite3_last_insert_rowid
+_sqlite3_interrupt
 _sqlite3_libversion
 _sqlite3_libversion_number
-_sqlite3_limit
-_sqlite3_malloc
-_sqlite3_malloc64
-_sqlite3_msize
 _sqlite3_open
 _sqlite3_open_v2
-_sqlite3_overload_function
 _sqlite3_prepare_v2
 _sqlite3_prepare_v3
-_sqlite3_preupdate_blobwrite
-_sqlite3_preupdate_count
-_sqlite3_preupdate_depth
-_sqlite3_preupdate_hook
-_sqlite3_preupdate_new
-_sqlite3_preupdate_old
-_sqlite3_progress_handler
-_sqlite3_randomness
-_sqlite3_realloc
-_sqlite3_realloc64
 _sqlite3_reset
-_sqlite3_reset_auto_extension
 _sqlite3_result_blob
 _sqlite3_result_double
 _sqlite3_result_error
@@ -98,106 +50,23 @@ _sqlite3_result_error_code
 _sqlite3_result_error_nomem
 _sqlite3_result_error_toobig
 _sqlite3_result_int
-_sqlite3_result_int64
 _sqlite3_result_null
-_sqlite3_result_pointer
-_sqlite3_result_subtype
 _sqlite3_result_text
-_sqlite3_result_zeroblob
-_sqlite3_result_zeroblob64
-_sqlite3_rollback_hook
-_sqlite3_serialize
-_sqlite3_set_authorizer
-_sqlite3_set_auxdata
-_sqlite3_set_last_insert_rowid
-_sqlite3_shutdown
 _sqlite3_sourceid
 _sqlite3_sql
-_sqlite3_status
-_sqlite3_status64
 _sqlite3_step
-_sqlite3_stmt_isexplain
-_sqlite3_stmt_readonly
-_sqlite3_stmt_status
 _sqlite3_strglob
-_sqlite3_stricmp
 _sqlite3_strlike
-_sqlite3_strnicmp
-_sqlite3_table_column_metadata
 _sqlite3_total_changes
 _sqlite3_total_changes64
-_sqlite3_trace_v2
-_sqlite3_txn_state
-_sqlite3_update_hook
-_sqlite3_uri_boolean
-_sqlite3_uri_int64
-_sqlite3_uri_key
-_sqlite3_uri_parameter
-_sqlite3_user_data
 _sqlite3_value_blob
 _sqlite3_value_bytes
 _sqlite3_value_double
-_sqlite3_value_dup
-_sqlite3_value_free
-_sqlite3_value_frombind
-_sqlite3_value_int
-_sqlite3_value_int64
-_sqlite3_value_nochange
-_sqlite3_value_numeric_type
-_sqlite3_value_pointer
-_sqlite3_value_subtype
 _sqlite3_value_text
 _sqlite3_value_type
 _sqlite3_vfs_find
 _sqlite3_vfs_register
-_sqlite3_vfs_unregister
-_sqlite3_vtab_collation
-_sqlite3_vtab_distinct
-_sqlite3_vtab_in
-_sqlite3_vtab_in_first
-_sqlite3_vtab_in_next
-_sqlite3_vtab_nochange
-_sqlite3_vtab_on_conflict
-_sqlite3_vtab_rhs_value
-_sqlite3changegroup_add
-_sqlite3changegroup_add_strm
-_sqlite3changegroup_delete
-_sqlite3changegroup_new
-_sqlite3changegroup_output
-_sqlite3changegroup_output_strm
-_sqlite3changeset_apply
-_sqlite3changeset_apply_strm
-_sqlite3changeset_apply_v2
-_sqlite3changeset_apply_v2_strm
-_sqlite3changeset_concat
-_sqlite3changeset_concat_strm
-_sqlite3changeset_conflict
-_sqlite3changeset_finalize
-_sqlite3changeset_fk_conflicts
-_sqlite3changeset_invert
-_sqlite3changeset_invert_strm
-_sqlite3changeset_new
-_sqlite3changeset_next
-_sqlite3changeset_old
-_sqlite3changeset_op
-_sqlite3changeset_pk
-_sqlite3changeset_start
-_sqlite3changeset_start_strm
-_sqlite3changeset_start_v2
-_sqlite3changeset_start_v2_strm
-_sqlite3session_attach
-_sqlite3session_changeset
-_sqlite3session_changeset_size
-_sqlite3session_changeset_strm
-_sqlite3session_config
-_sqlite3session_create
-_sqlite3session_delete
-_sqlite3session_diff
-_sqlite3session_enable
-_sqlite3session_indirect
-_sqlite3session_isempty
-_sqlite3session_memory_used
-_sqlite3session_object_config
-_sqlite3session_patchset
-_sqlite3session_patchset_strm
-_sqlite3session_table_filter
+_sqlite3_wasm_db_error
+_sqlite3_wasm_enum_json
+_malloc
+_free

ext/wasm/api/README.md

@@ -83,15 +83,18 @@ browser client:
   helpers for use by downstream code which creates `sqlite3_vfs`
   and `sqlite3_module` implementations.
 - **`sqlite3-vfs-opfs.c-pp.js`**\  
-  is an sqlite3 VFS implementation which supports Google Chrome's
-  Origin-Private FileSystem (OPFS) as a storage layer to provide
-  persistent storage for database files in a browser. It requires...
+  is an sqlite3 VFS implementation which supports the Origin-Private
+  FileSystem (OPFS) as a storage layer to provide persistent storage
+  for database files in a browser. It requires...
     - **`sqlite3-opfs-async-proxy.js`**\  
       is the asynchronous backend part of the OPFS proxy. It speaks
       directly to the (async) OPFS API and channels those results back
       to its synchronous counterpart. This file, because it must be
       started in its own Worker, is not part of the amalgamation.
-- **`api/sqlite3-api-cleanup.js`**\  
+- **`sqlite3-vfs-opfs-sahpool.c-pp.js`**\  
+  is another sqlite3 VFS supporting the OPFS, but uses a completely
+  different approach that the above-listed one.
+- **`sqlite3-api-cleanup.js`**\  
   The previous files do not immediately extend the library. Instead
   they add callback functions to be called during its
   bootstrapping. Some also temporarily create global objects in order
@@ -108,13 +111,15 @@ browser client:
 with `c-pp`](#c-pp), noting that such preprocessing may be applied
 after all of the relevant files are concatenated. That extension is
 used primarily to keep the code maintainers cognisant of the fact that
-those files contain constructs which will not run as-is in JavaScript.
+those files contain constructs which may not run as-is in any given
+JavaScript environment.
 
 The build process glues those files together, resulting in
-`sqlite3-api.js`, which is everything except for the `post-js-*.js`
-files, and `sqlite3.js`, which is the Emscripten-generated amalgamated
-output and includes the `post-js-*.js` parts, as well as the
-Emscripten-provided module loading pieces.
+`sqlite3-api.js`, which is everything except for the
+`pre/post-js-*.js` files, and `sqlite3.js`, which is the
+Emscripten-generated amalgamated output and includes the
+`pre/post-js-*.js` parts, as well as the Emscripten-provided module
+loading pieces.
 
 The non-JS outlier file is `sqlite3-wasm.c`: it is a proxy for
 `sqlite3.c` which `#include`'s that file and adds a couple more
@@ -152,8 +157,8 @@ Preprocessing of Source Files
 ------------------------------------------------------------------------
 
 Certain files in the build require preprocessing to filter in/out
-parts which differ between vanilla JS builds and ES6 Module
-(a.k.a. esm) builds. The preprocessor application itself is in
+parts which differ between vanilla JS, ES6 Modules, and node.js
+builds. The preprocessor application itself is in
 [`c-pp.c`](/file/ext/wasm/c-pp.c) and the complete technical details
 of such preprocessing are maintained in
 [`GNUMakefile`](/file/ext/wasm/GNUmakefile).

ext/wasm/api/extern-post-js.c-pp.js

@@ -62,6 +62,7 @@ const toExportForESM =
   globalThis.sqlite3InitModule = function ff(...args){
     //console.warn("Using replaced sqlite3InitModule()",globalThis.location);
     return originalInit(...args).then((EmscriptenModule)=>{
+//#if wasmfs
       if('undefined'!==typeof WorkerGlobalScope &&
          EmscriptenModule['ENVIRONMENT_IS_PTHREAD']){
         /** Workaround for wasmfs-generated worker, which calls this
@@ -72,6 +73,7 @@ const toExportForESM =
         //console.warn("sqlite3InitModule() returning E-module.",EmscriptenModule);
         return EmscriptenModule;
       }
+//#endif
       //console.warn("sqlite3InitModule() returning sqlite3 object.");
       const s = EmscriptenModule.sqlite3;
       s.scriptInfo = initModuleState;
@@ -119,5 +121,6 @@ const toExportForESM =
   return globalThis.sqlite3InitModule /* required for ESM */;
 })();
 //#if target=es6-module
-export { toExportForESM as default, toExportForESM as sqlite3InitModule }
+sqlite3InitModule = toExportForESM;
+export default sqlite3InitModule;
 //#endif

ext/wasm/api/sqlite3-api-cleanup.js

@@ -22,7 +22,9 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build
   */
   const SABC = Object.assign(
     Object.create(null), {
-      exports: Module['asm'],
+      exports: ('undefined'===typeof wasmExports)
+        ? Module['asm']/* emscripten <=3.1.43 */
+        : wasmExports  /* emscripten >=3.1.44 */,
       memory: Module.wasmMemory /* gets set if built with -sIMPORTED_MEMORY */
     },
     globalThis.sqlite3ApiConfig || {}

ext/wasm/api/sqlite3-api-glue.js

@@ -728,6 +728,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
        Populate api object with sqlite3_...() by binding the "raw" wasm
        exports into type-converting proxies using wasm.xWrap().
     */
+    if(0 === wasm.exports.sqlite3_step.length){
+      /* This environment wraps exports in nullary functions, which means
+         we must disable the arg-count validation we otherwise perform
+         on the wrappers. */
+      wasm.xWrap.doArgcCheck = false;
+      sqlite3.config.warn(
+        "Disabling sqlite3.wasm.xWrap.doArgcCheck due to environmental quirks."
+      );
+    }
     for(const e of wasm.bindingSignatures){
       capi[e[0]] = wasm.xWrap.apply(null, e);
     }

ext/wasm/api/sqlite3-api-prologue.js

@@ -88,9 +88,9 @@
      can be replaced with (e.g.) empty functions to squelch all such
      output.
 
-   - `wasmfsOpfsDir`[^1]: As of 2022-12-17, this feature does not
-     currently work due to incompatible Emscripten-side changes made
-     in the WASMFS+OPFS combination. This option is currently ignored.
+   - `wasmfsOpfsDir`[^1]: Specifies the "mount point" of the OPFS-backed
+     filesystem in WASMFS-capable builds.
+
 
    [^1] = This property may optionally be a function, in which case
           this function calls that function to fetch the value,
@@ -125,11 +125,11 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
     log: console.log.bind(console),
     wasmfsOpfsDir: '/opfs',
     /**
-       useStdAlloc is just for testing an allocator discrepancy. The
+       useStdAlloc is just for testing allocator discrepancies. The
        docs guarantee that this is false in the canonical builds. For
        99% of purposes it doesn't matter which allocators we use, but
-       it becomes significant with, e.g., sqlite3_deserialize()
-       and certain wasm.xWrap.resultAdapter()s.
+       it becomes significant with, e.g., sqlite3_deserialize() and
+       certain wasm.xWrap.resultAdapter()s.
     */
     useStdAlloc: false
   }, apiConfig || {});
@@ -1645,7 +1645,8 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
          do not.
       */
       tgt.push(capi.sqlite3_value_to_js(
-        wasm.peekPtr(pArgv + (wasm.ptrSizeof * i))
+        wasm.peekPtr(pArgv + (wasm.ptrSizeof * i)),
+        throwIfCannotConvert
       ));
     }
     return tgt;
@@ -1861,6 +1862,9 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
     client: undefined,
 
     /**
+       This function is not part of the public interface, but a
+       piece of internal bootstrapping infrastructure.
+
        Performs any optional asynchronous library-level initialization
        which might be required. This function returns a Promise which
        resolves to the sqlite3 namespace object. Any error in the
@@ -1876,27 +1880,19 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
        then it must be called by client-level code, which must not use
        the library until the returned promise resolves.
 
-       Bug: if called while a prior call is still resolving, the 2nd
-       call will resolve prematurely, before the 1st call has finished
-       resolving. The current build setup precludes that possibility,
-       so it's only a hypothetical problem if/when this function
-       ever needs to be invoked by clients.
+       If called multiple times it will return the same promise on
+       subsequent calls. The current build setup precludes that
+       possibility, so it's only a hypothetical problem if/when this
+       function ever needs to be invoked by clients.
 
        In Emscripten-based builds, this function is called
        automatically and deleted from this object.
     */
-    asyncPostInit: async function(){
-      let lip = sqlite3ApiBootstrap.initializersAsync;
+    asyncPostInit: async function ff(){
+      if(ff.isReady instanceof Promise) return ff.isReady;
+      let lia = sqlite3ApiBootstrap.initializersAsync;
       delete sqlite3ApiBootstrap.initializersAsync;
-      if(!lip || !lip.length) return Promise.resolve(sqlite3);
-      lip = lip.map((f)=>{
-        const p = (f instanceof Promise) ? f : f(sqlite3);
-        return p.catch((e)=>{
-          console.error("an async sqlite3 initializer failed:",e);
-          throw e;
-        });
-      });
-      const postInit = ()=>{
+      const postInit = async ()=>{
         if(!sqlite3.__isUnderTest){
           /* Delete references to internal-only APIs which are used by
              some initializers. Retain them when running in test mode
@@ -1905,23 +1901,25 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
           /* It's conceivable that we might want to expose
              StructBinder to client-side code, but it's only useful if
              clients build their own sqlite3.wasm which contains their
-             one C struct types. */
+             own C struct types. */
           delete sqlite3.StructBinder;
         }
         return sqlite3;
       };
-      if(1){
-        /* Run all initializers in sequence. The advantage is that it
-           allows us to have post-init cleanup defined outside of this
-           routine at the end of the list and have it run at a
-           well-defined time. */
-        let p = lip.shift();
-        while(lip.length) p = p.then(lip.shift());
-        return p.then(postInit);
-      }else{
-        /* Run them in an arbitrary order. */
-        return Promise.all(lip).then(postInit);
+      const catcher = (e)=>{
+        config.error("an async sqlite3 initializer failed:",e);
+        throw e;
+      };
+      if(!lia || !lia.length){
+        return ff.isReady = postInit().catch(catcher);
       }
+      lia = lia.map((f)=>{
+        return (f instanceof Function) ? async x=>f(sqlite3) : f;
+      });
+      lia.push(postInit);
+      let p = Promise.resolve(sqlite3);
+      while(lia.length) p = p.then(lia.shift());
+      return ff.isReady = p.catch(catcher);
     },
     /**
        scriptInfo ideally gets injected into this object by the
@@ -1981,7 +1979,7 @@ globalThis.sqlite3ApiBootstrap.initializers = [];
   specifically for initializers which are asynchronous. All entries in
   this list must be either async functions, non-async functions which
   return a Promise, or a Promise. Each function in the list is called
-  with the sqlite3 ojbect as its only argument.
+  with the sqlite3 object as its only argument.
 
   The resolved value of any Promise is ignored and rejection will kill
   the asyncPostInit() process (at an indeterminate point because all

ext/wasm/api/sqlite3-opfs-async-proxy.js

@@ -35,6 +35,9 @@
 
   https://developer.chrome.com/blog/sync-methods-for-accesshandles/
 
+  Firefox v111 and Safari 16.4, both released in March 2023, also
+  include this.
+
   We cannot change to the sync forms at this point without breaking
   clients who use Chrome v104-ish or higher. truncate(), getSize(),
   flush(), and close() are now (as of v108) synchronous. Calling them

ext/wasm/api/sqlite3-v-helper.js

@@ -100,7 +100,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
 
      ACHTUNG: because we cannot generically know how to transform JS
      exceptions into result codes, the installed functions do no
-     automatic catching of exceptions. It is critical, to avoid 
+     automatic catching of exceptions. It is critical, to avoid
      undefined behavior in the C layer, that methods mapped via
      this function do not throw. The exception, as it were, to that
      rule is...
@@ -295,7 +295,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
 
      - If `struct.$zName` is falsy and the entry has a string-type
        `name` property, `struct.$zName` is set to the C-string form of
-       that `name` value before registerVfs() is called.
+       that `name` value before registerVfs() is called. That string
+       gets added to the on-dispose state of the struct.
 
      On success returns this object. Throws on error.
   */
@@ -608,7 +609,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
      This is to facilitate creation of those methods inline in the
      passed-in object without requiring the client to explicitly get a
      reference to one of them in order to assign it to the other
-     one. 
+     one.
 
      The `catchExceptions`-installed handlers will account for
      identical references to the above functions and will install the

ext/wasm/api/sqlite3-vfs-opfs.c-pp.js

@@ -1,3 +1,4 @@
+//#ifnot target=node
 /*
   2022-09-18
 
@@ -23,7 +24,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
    installOpfsVfs() returns a Promise which, on success, installs an
    sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs
    which accept a VFS. It is intended to be called via
-   sqlite3ApiBootstrap.initializersAsync or an equivalent mechanism.
+   sqlite3ApiBootstrap.initializers or an equivalent mechanism.
 
    The installed VFS uses the Origin-Private FileSystem API for
    all file storage. On error it is rejected with an exception
@@ -101,6 +102,10 @@ const installOpfsVfs = function callee(options){
     options = Object.create(null);
   }
   const urlParams = new URL(globalThis.location.href).searchParams;
+  if(urlParams.has('opfs-disable')){
+    //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.');
+    return Promise.resolve(sqlite3);
+  }
   if(undefined===options.verbose){
     options.verbose = urlParams.has('opfs-verbose')
       ? (+urlParams.get('opfs-verbose') || 2) : 1;
@@ -200,9 +205,9 @@ const installOpfsVfs = function callee(options){
       opfsVfs.dispose();
       return promiseReject_(err);
     };
-    const promiseResolve = (value)=>{
+    const promiseResolve = ()=>{
       promiseWasRejected = false;
-      return promiseResolve_(value);
+      return promiseResolve_(sqlite3);
     };
     const W =
 //#if target=es6-bundler-friendly
@@ -236,6 +241,7 @@ const installOpfsVfs = function callee(options){
           ? new sqlite3_vfs(pDVfs)
           : null /* dVfs will be null when sqlite3 is built with
                     SQLITE_OS_OTHER. */;
+    opfsIoMethods.$iVersion = 1;
     opfsVfs.$iVersion = 2/*yes, two*/;
     opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof;
     opfsVfs.$mxPathname = 1024/*sure, why not?*/;
@@ -1321,10 +1327,10 @@ const installOpfsVfs = function callee(options){
                   sqlite3.opfs = opfsUtil;
                   opfsUtil.rootDirectory = d;
                   log("End of OPFS sqlite3_vfs setup.", opfsVfs);
-                  promiseResolve(sqlite3);
+                  promiseResolve();
                 }).catch(promiseReject);
               }else{
-                promiseResolve(sqlite3);
+                promiseResolve();
               }
             }catch(e){
               error(e);
@@ -1361,7 +1367,10 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{
     });
   }catch(e){
     sqlite3.config.error("installOpfsVfs() exception:",e);
-    throw e;
+    return Promise.reject(e);
   }
 });
 }/*sqlite3ApiBootstrap.initializers.push()*/);
+//#else
+/* The OPFS VFS parts are elided from builds targeting node.js. */
+//#endif target=node

ext/wasm/api/sqlite3-wasm.c

@@ -151,7 +151,7 @@
 /**********************************************************************/
 /* SQLITE_T... */
 #ifndef SQLITE_TEMP_STORE
-# define SQLITE_TEMP_STORE 3
+# define SQLITE_TEMP_STORE 2
 #endif
 #ifndef SQLITE_THREADSAFE
 # define SQLITE_THREADSAFE 0

ext/wasm/api/sqlite3-worker1-promiser.c-pp.js

@@ -114,7 +114,7 @@
    by all client code except that which tests this API. The `row`
    property contains the row result in the form implied by the
    `rowMode` option (defaulting to `'array'`). The `rowNumber` is a
-   1-based integer value incremented by 1 on each call into the
+   1-based integer value incremented by 1 on each call into th 
    callback.
 
    At the end of the result set, the same event is fired with
@@ -122,17 +122,8 @@
    the end of the result set has been reached. Note that the rows
    arrive via worker-posted messages, with all the implications
    of that.
-
-   Notable shortcomings:
-
-   - This API was not designed with ES6 modules in mind. Neither Firefox
-     nor Safari support, as of March 2023, the {type:"module"} flag to the
-     Worker constructor, so that particular usage is not something we're going
-     to target for the time being:
-
-     https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker
 */
-globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){
+self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){
   // Inspired by: https://stackoverflow.com/a/52439530
   if(1===arguments.length && 'function'===typeof arguments[0]){
     const f = config;
@@ -169,7 +160,7 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi
       if(msgHandler && msgHandler.onrow){
         msgHandler.onrow(ev);
         return;
-      }
+      }        
       if(config.onunhandled) config.onunhandled(arguments[0]);
       else err("sqlite3Worker1Promiser() unhandled worker message:",ev);
       return;
@@ -196,9 +187,10 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi
     if(1===arguments.length){
       msg = arguments[0];
     }else if(2===arguments.length){
-      msg = Object.create(null);
-      msg.type = arguments[0];
-      msg.args = arguments[1];
+      msg = {
+        type: arguments[0],
+        args: arguments[1]
+      };
     }else{
       toss("Invalid arugments for sqlite3Worker1Promiser()-created factory.");
     }
@@ -244,31 +236,28 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi
     return p;
   };
 }/*sqlite3Worker1Promiser()*/;
-globalThis.sqlite3Worker1Promiser.defaultConfig = {
+self.sqlite3Worker1Promiser.defaultConfig = {
   worker: function(){
 //#if target=es6-bundler-friendly
-    return new Worker("sqlite3-worker1-bundler-friendly.mjs",{
-      type: 'module' /* Noting that neither Firefox nor Safari suppor this,
-                        as of this writing. */
-    });
+    return new Worker("sqlite3-worker1.js");
 //#else
     let theJs = "sqlite3-worker1.js";
     if(this.currentScript){
       const src = this.currentScript.src.split('/');
       src.pop();
       theJs = src.join('/')+'/' + theJs;
-      //sqlite3.config.warn("promiser currentScript, theJs =",this.currentScript,theJs);
-    }else if(globalThis.location){
-      //sqlite3.config.warn("promiser globalThis.location =",globalThis.location);
-      const urlParams = new URL(globalThis.location.href).searchParams;
+      //console.warn("promiser currentScript, theJs =",this.currentScript,theJs);
+    }else{
+      //console.warn("promiser self.location =",self.location);
+      const urlParams = new URL(self.location.href).searchParams;
       if(urlParams.has('sqlite3.dir')){
         theJs = urlParams.get('sqlite3.dir') + '/' + theJs;
       }
     }
-    return new Worker(theJs + globalThis.location.search);
+    return new Worker(theJs + self.location.search);
 //#endif
   }.bind({
-    currentScript: globalThis?.document?.currentScript
+    currentScript: self?.document?.currentScript
   }),
   onerror: (...args)=>console.error('worker1 promiser error',...args)
 };

ext/wasm/common/testing.css

@@ -40,8 +40,41 @@ span.labeled-input {
 .tests-pass { background-color: green; color: white }
 .tests-fail { background-color: red; color: yellow }
 .faded { opacity: 0.5; }
-.group-start { color: blue; }
-.group-end { color: blue; }
+.group-start {
+  color: blue;
+  background-color: skyblue;
+  font-weight: bold;
+  border-top: 1px dotted blue;
+  padding: 0.5em;
+  margin-top: 0.5em;
+}
+.group-end {
+  padding: 0.5em;
+  margin-bottom: 0.25em;
+  /*border-bottom: 1px dotted blue;*/
+}
+.group-end.green {
+  background: lightgreen;
+  border-bottom: 1px dotted green;
+}
+.one-test-line, .skipping-group {
+  margin-left: 3em;
+}
+.skipping-test, .skipping-group {
+  padding: 0.25em 0.5em;
+  background-color: #ffff73;
+}
+.skipping-test {
+  margin-left: 6em;
+}
+.one-test-summary {
+  margin-left: 6em;
+}
+.full-test-summary {
+  padding-bottom: 0.5em;
+  padding-top: 0.5em;
+  border-top: 1px solid black;
+}
 .input-wrapper {
   white-space: nowrap;
   display: flex;

ext/wasm/common/whwasmutil.js

@@ -111,7 +111,8 @@
      of `target.instance` (a WebAssembly.Module instance) and it must
      contain the symbols exported by the WASM module associated with
      this code. In an Enscripten environment it must be set to
-     `Module['asm']`. The exports object must contain a minimum of the
+     `Module['asm']` (versions <=3.1.43) or `wasmExports` (versions
+     >=3.1.44). The exports object must contain a minimum of the
      following symbols:
 
      - `memory`: a WebAssembly.Memory object representing the WASM
@@ -612,8 +613,6 @@ globalThis.WhWasmUtilInstaller = function(target){
   target.installFunction = (func, sig)=>__installFunction(func, sig, false);
 
   /**
-     EXPERIMENTAL! DO NOT USE IN CLIENT CODE!
-
      Works exactly like installFunction() but requires that a
      scopedAllocPush() is active and uninstalls the given function
      when that alloc scope is popped via scopedAllocPop().
@@ -1179,7 +1178,7 @@ globalThis.WhWasmUtilInstaller = function(target){
     cache.scopedAlloc.splice(n,1);
     for(let p; (p = state.pop()); ){
       if(target.functionEntry(p)){
-        //console.warn("scopedAllocPop() uninstalling transient function",p);
+        //console.warn("scopedAllocPop() uninstalling function",p);
         target.uninstallFunction(p);
       }
       else target.dealloc(p);
@@ -1636,6 +1635,7 @@ globalThis.WhWasmUtilInstaller = function(target){
                      'and is not intended to be invoked from',
                      'client-level code. Invoked with:',opt);
       }
+      this.name = opt.name || "unnamed";
       this.signature = opt.signature;
       if(opt.contextKey instanceof Function){
         this.contextKey = opt.contextKey;
@@ -1697,14 +1697,16 @@ globalThis.WhWasmUtilInstaller = function(target){
        exactly the 2nd and 3rd arguments are.
     */
     convertArg(v,argv,argIndex){
-      //FuncPtrAdapter.debugOut("FuncPtrAdapter.convertArg()",this.signature,this.transient,v);
+      //FuncPtrAdapter.debugOut("FuncPtrAdapter.convertArg()",this.name,this.signature,this.transient,v);
       let pair = this.singleton;
       if(!pair && this.isContext){
         pair = this.contextMap(this.contextKey(argv,argIndex));
+        //FuncPtrAdapter.debugOut(this.name, this.signature, "contextKey() =",this.contextKey(argv,argIndex), pair);
       }
       if(pair && pair[0]===v) return pair[1];
       if(v instanceof Function){
         /* Install a WASM binding and return its pointer. */
+        //FuncPtrAdapter.debugOut("FuncPtrAdapter.convertArg()",this.name,this.signature,this.transient,v,pair);
         if(this.callProxy) v = this.callProxy(v);
         const fp = __installFunction(v, this.signature, this.isTransient);
         if(FuncPtrAdapter.debugFuncInstall){
@@ -1718,7 +1720,18 @@ globalThis.WhWasmUtilInstaller = function(target){
               FuncPtrAdapter.debugOut("FuncPtrAdapter uninstalling", this,
                                       this.contextKey(argv,argIndex), '@'+pair[1], v);
             }
-            try{target.uninstallFunction(pair[1])}
+            try{
+              /* Because the pending native call might rely on the
+                 pointer we're replacing, e.g. as is normally the case
+                 with sqlite3's xDestroy() methods, we don't
+                 immediately uninstall but instead add its pointer to
+                 the scopedAlloc stack, which will be cleared when the
+                 xWrap() mechanism is done calling the native
+                 function. We're relying very much here on xWrap()
+                 having pushed an alloc scope.
+              */
+              cache.scopedAlloc[cache.scopedAlloc.length-1].push(pair[1]);
+            }
             catch(e){/*ignored*/}
           }
           pair[0] = v;
@@ -1726,13 +1739,14 @@ globalThis.WhWasmUtilInstaller = function(target){
         }
         return fp;
       }else if(target.isPtr(v) || null===v || undefined===v){
+        //FuncPtrAdapter.debugOut("FuncPtrAdapter.convertArg()",this.name,this.signature,this.transient,v,pair);
         if(pair && pair[1] && pair[1]!==v){
           /* uninstall stashed mapping and replace stashed mapping with v. */
           if(FuncPtrAdapter.debugFuncInstall){
             FuncPtrAdapter.debugOut("FuncPtrAdapter uninstalling", this,
                                     this.contextKey(argv,argIndex), '@'+pair[1], v);
           }
-          try{target.uninstallFunction(pair[1])}
+          try{ cache.scopedAlloc[cache.scopedAlloc.length-1].push(pair[1]) }
           catch(e){/*ignored*/}
           pair[0] = pair[1] = (v | 0);
         }

ext/wasm/demo-123.js

@@ -235,10 +235,10 @@
 
        - get change count (total or statement-local, 32- or 64-bit)
        - get a DB's file name
-    
+
        Misc. Stmt features:
 
-       - Various forms of bind() 
+       - Various forms of bind()
        - clearBindings()
        - reset()
        - Various forms of step()

ext/wasm/dist.make

@@ -38,7 +38,7 @@ dist-name := $(dist-name-prefix)-TEMP
 # date. Our general policy is that we want the smallest binaries for
 # dist zip files, so use the oz build unless there is a compelling
 # reason not to.
-dist.build ?= qoz
+dist.build ?= oz
 
 dist-dir.top := $(dist-name)
 dist-dir.jswasm := $(dist-dir.top)/$(notdir $(dir.dout))

ext/wasm/fiddle.make

@@ -18,7 +18,7 @@ endif
 ifeq (,$(SHELL_SRC))
 $(error Could not parse SHELL_SRC from $(dir.top)/Makefile.)
 endif
-$(dir.top)/shell.c: $(SHELL_SRC) $(dir.top)/tool/mkshellc.tcl
+$(dir.top)/shell.c: $(SHELL_SRC) $(dir.top)/tool/mkshellc.tcl $(sqlite3.c)
 	$(MAKE) -C $(dir.top) shell.c
 # /shell.c
 ########################################################################
@@ -61,9 +61,9 @@ $(fiddle.SOAP.js): $(SOAP.js)
 $(eval $(call call-make-pre-post,fiddle-module,vanilla))
 $(fiddle-module.js): $(MAKEFILE) $(MAKEFILE.fiddle) \
     $(EXPORTED_FUNCTIONS.fiddle) \
-    $(fiddle.cses) $(pre-post-fiddle-module.deps.vanilla) $(fiddle.SOAP.js)
+    $(fiddle.cses) $(pre-post-fiddle-module-vanilla.deps) $(fiddle.SOAP.js)
 	$(emcc.bin) -o $@ $(fiddle.emcc-flags) \
-    $(pre-post-fiddle-module.flags.vanilla) \
+    $(pre-post-fiddle-module-vanilla.flags) \
     $(fiddle.cses)
 	$(maybe-wasm-strip) $(fiddle-module.wasm)
 	gzip < $@ > $@.gz

ext/wasm/fiddle/fiddle-worker.js

@@ -221,7 +221,7 @@
       f._();
     }
   };
-  
+
   self.onmessage = function f(ev){
     ev = ev.data;
     if(!f.cache){
@@ -371,12 +371,14 @@
   sqlite3InitModule(fiddleModule).then((_sqlite3)=>{
     sqlite3 = _sqlite3;
     console.warn("Installing sqlite3 module globally (in Worker)",
-                 "for use in the dev console.");
-    self.sqlite3 = sqlite3;
+                 "for use in the dev console.", sqlite3);
+    globalThis.sqlite3 = sqlite3;
     const dbVfs = sqlite3.wasm.xWrap('fiddle_db_vfs', "*", ['string']);
     fiddleModule.fsUnlink = (fn)=>{
       return sqlite3.wasm.sqlite3_wasm_vfs_unlink(dbVfs(0), fn);
     };
     wMsg('fiddle-ready');
-  })/*then()*/;
+  }).catch(e=>{
+    console.error("Fiddle worker init failed:",e);
+  });
 })();

ext/wasm/fiddle/fiddle.html

@@ -273,6 +273,11 @@
         </fieldset>
       </div>
     </div> <!-- #view-split -->
+    <!-- Maintenance notes:
+
+        ... TODO... currently being refactored...
+
+    -->
     <script src="fiddle.js"></script>
   </body>
 </html>

ext/wasm/index.html

@@ -96,6 +96,10 @@
             <li><a href='speedtest1-worker.html?size=15'>speedtest1-worker</a>: an interactive Worker-thread variant of speedtest1.</li>
             <li><a href='speedtest1-worker.html?vfs=opfs&size=10'>speedtest1-worker?vfs=opfs</a>: speedtest1-worker with the
               OPFS VFS preselected and configured for a moderate workload.</li>
+            <li><a href='speedtest1-worker.html?vfs=opfs-sahpool&size=10'>speedtest1-worker?vfs=opfs-sahpool</a>:
+              speedtest1-worker with the OPFS-SAHPOOL VFS preselected
+              and configured for a moderate workload.
+            </li>
           </ul>
         </li>
         <li>The obligatory "misc." category...
@@ -103,7 +107,8 @@
             <li><a href='module-symbols.html'>module-symbols</a> gives
               a high-level overview of the symbols exposed by the JS
               module.</li>
-            <li><a href='batch-runner.html'>batch-runner</a>: runs batches of SQL exported from speedtest1.</li>
+            <!--li><a href='batch-runner.html'>batch-runner</a>: runs batches of
+                SQL exported from speedtest1.</li-->
             <li><a href='test-opfs-vfs.html'>test-opfs-vfs</a>
               (<a href='test-opfs-vfs.html?opfs-sanity-check&opfs-verbose'>same
               with verbose output and sanity-checking tests</a>) is an

ext/wasm/speedtest1-worker.html

@@ -171,7 +171,8 @@
     const urlParams = new URL(self.location.href).searchParams;
     const W = new Worker(
         "speedtest1-worker.js?sqlite3.dir=jswasm"+
-            (urlParams.has('opfs-verbose') ? '&opfs-verbose' : '')
+            (urlParams.has('opfs-verbose') ? '&opfs-verbose' : '')+
+            (urlParams.has('opfs-disable') ? '&opfs-disable' : '')
     );
     const mPost = function(msgType,payload){
         W.postMessage({type: msgType, data: payload});
@@ -267,7 +268,7 @@
         if(urlParams.has('flags')){
             preselectedFlags.push(...urlParams.get('flags').split(','));
         }
-        if('opfs'!==urlParams.get('vfs')){
+        if(!urlParams.get('vfs')){
             preselectedFlags.push('--memdb');
         }
         Object.keys(flags).sort().forEach(function(f){

ext/wasm/speedtest1-worker.js

@@ -5,7 +5,7 @@
   if(urlParams.has('sqlite3.dir')){
     speedtestJs = urlParams.get('sqlite3.dir') + '/' + speedtestJs;
   }
-  importScripts('common/whwasmutil.js', speedtestJs);
+  importScripts(speedtestJs);
   /**
      If this environment contains OPFS, this function initializes it and
      returns the name of the dir on which OPFS is mounted, else it returns
@@ -48,7 +48,7 @@
   const log = (...args)=>logMsg('stdout',args);
   const logErr = (...args)=>logMsg('stderr',args);
 
-  const runSpeedtest = function(cliFlagsArray){
+  const runSpeedtest = async function(cliFlagsArray){
     const scope = App.wasm.scopedAllocPush();
     const dbFile = App.pDir+"/speedtest1.sqlite3";
     try{
@@ -56,7 +56,28 @@
         "speedtest1.wasm", ...cliFlagsArray, dbFile
       ];
       App.logBuffer.length = 0;
+      const ndxSahPool = argv.indexOf('opfs-sahpool');
+      const realSahName = 'opfs-sahpool-speedtest1';
+      if(ndxSahPool>0){
+        argv[ndxSahPool] = realSahName;
+        log("Updated argv for opfs-sahpool: --vfs",realSahName);
+      }
       mPost('run-start', [...argv]);
+      if(App.sqlite3.installOpfsSAHPoolVfs
+         && !App.sqlite3.$SAHPoolUtil
+         && ndxSahPool>0){
+        log("Installing opfs-sahpool as",realSahName,"...");
+        await App.sqlite3.installOpfsSAHPoolVfs({
+          name: realSahName,
+          initialCapacity: 3,
+          clearOnInit: true,
+          verbosity: 2
+        }).then(PoolUtil=>{
+          log("opfs-sahpool successfully installed as",realSahName);
+          App.sqlite3.$SAHPoolUtil = PoolUtil;
+          //console.log("sqlite3.oo1.OpfsSAHPoolDb =", App.sqlite3.oo1.OpfsSAHPoolDb);
+        });
+      }
       App.wasm.xCall('wasm_main', argv.length,
                      App.wasm.scopedAllocMainArgv(argv));
     }catch(e){
@@ -71,20 +92,38 @@
   self.onmessage = function(msg){
     msg = msg.data;
     switch(msg.type){
-        case 'run': runSpeedtest(msg.data || []); break;
+        case 'run':
+          runSpeedtest(msg.data || [])
+            .catch(e=>mPost('error',e));
+          break;
         default:
           logErr("Unhandled worker message type:",msg.type);
           break;
     }
   };
 
+  const sahpSanityChecks = function(sqlite3){
+    log("Attempting OpfsSAHPoolDb sanity checks...");
+    const db = new sqlite3.oo1.OpfsSAHPoolDb('opfs-sahpoool.db');
+    const fn = db.filename;
+    db.exec([
+      'create table t(a);',
+      'insert into t(a) values(1),(2),(3);'
+    ]);
+    db.close();
+    sqlite3.wasm.sqlite3_wasm_vfs_unlink(sqlite3_vfs_find("opfs-sahpool"), fn);
+    log("SAH sanity checks done.");
+  };
+
   const EmscriptenModule = {
     print: log,
     printErr: logErr,
     setStatus: (text)=>mPost('load-status',text)
   };
-  self.sqlite3InitModule(EmscriptenModule).then((sqlite3)=>{
-    const S = sqlite3;
+  log("Initializing speedtest1 module...");
+  self.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{
+    const S = globalThis.S = App.sqlite3 = sqlite3;
+    log("Loaded speedtest1 module. Setting up...");
     App.vfsUnlink = function(pDb, fname){
       const pVfs = S.wasm.sqlite3_wasm_db_vfs(pDb, 0);
       if(pVfs) S.wasm.sqlite3_wasm_vfs_unlink(pVfs, fname||0);
@@ -95,5 +134,7 @@
     //else log("Using transient storage.");
     mPost('ready',true);
     log("Registered VFSes:", ...S.capi.sqlite3_js_vfs_list());
+  }).catch(e=>{
+    logErr(e);
   });
 })();

ext/wasm/tester1.c-pp.js

@@ -65,6 +65,14 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
   const haveWasmCTests = ()=>{
     return !!wasm.exports.sqlite3_wasm_test_intptr;
   };
+  const hasOpfs = ()=>{
+    return globalThis.FileSystemHandle
+      && globalThis.FileSystemDirectoryHandle
+      && globalThis.FileSystemFileHandle
+      && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle
+      && navigator?.storage?.getDirectory;
+  };
+
   {
     const mapToString = (v)=>{
       switch(typeof v){
@@ -159,8 +167,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
     /** Running total of the number of tests run via
         this API. */
     counter: 0,
-    /* Separator line for log messages. */
-    separator: '------------------------------------------------------------',
     /**
        If expr is a function, it is called and its result
        is returned, coerced to a bool, else expr, coerced to
@@ -248,13 +254,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
           return this;
         },
         run: async function(sqlite3){
-          log(TestUtil.separator);
           logClass('group-start',"Group #"+this.number+':',this.name);
-          const indent = '    ';
           if(this.predicate){
             const p = this.predicate(sqlite3);
             if(!p || 'string'===typeof p){
-              logClass('warning',indent,
+              logClass(['warning','skipping-group'],
                        "SKIPPING group:", p ? p : "predicate says to" );
               return;
             }
@@ -266,26 +270,34 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
           for(const t of this.tests){
             ++i;
             const n = this.number+"."+i;
-              log(indent, n+":", t.name);
+            logClass('one-test-line', n+":", t.name);
             if(t.predicate){
               const p = t.predicate(sqlite3);
               if(!p || 'string'===typeof p){
-                logClass('warning',indent,
+                logClass(['warning','skipping-test'],
                          "SKIPPING:", p ? p : "predicate says to" );
                 skipped.push( n+': '+t.name );
                 continue;
               }
             }
             const tc = TestUtil.counter, now = performance.now();
-            await t.test.call(groupState, sqlite3);
+            let rc = t.test.call(groupState, sqlite3);
+            /*if(rc instanceof Promise){
+              rc = rc.catch((e)=>{
+                error("Test failure:",e);
+                throw e;
+              });
+            }*/
+            await rc;
             const then = performance.now();
             runtime += then - now;
-            logClass('faded',indent, indent,
+            logClass(['faded','one-test-summary'],
                      TestUtil.counter - tc, 'assertion(s) in',
                      roundMs(then-now),'ms');
           }
-          logClass('green',
-                   "Group #"+this.number+":",(TestUtil.counter - assertCount),
+          logClass(['green','group-end'],
+                   "#"+this.number+":",
+                   (TestUtil.counter - assertCount),
                    "assertion(s) in",roundMs(runtime),"ms");
           if(0 && skipped.length){
             logClass('warning',"SKIPPED test(s) in group",this.number+":",skipped);
@@ -321,8 +333,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
             await g.run(sqlite3);
             runtime += performance.now() - now;
           }
-          log(TestUtil.separator);
-          logClass(['strong','green'],
+          logClass(['strong','green','full-test-summary'],
                    "Done running tests.",TestUtil.counter,"assertions in",
                    roundMs(runtime),'ms');
           pok();
@@ -339,6 +350,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
   T.g = T.addGroup;
   T.t = T.addTest;
   let capi, wasm/*assigned after module init*/;
+  const sahPoolConfig = {
+    name: 'opfs-sahpool-tester1',
+    clearOnInit: true,
+    initialCapacity: 6
+  };
   ////////////////////////////////////////////////////////////////////////
   // End of infrastructure setup. Now define the tests...
   ////////////////////////////////////////////////////////////////////////
@@ -1288,7 +1304,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
       if(1){
         const vfsList = capi.sqlite3_js_vfs_list();
         T.assert(vfsList.length>1);
-        //log("vfsList =",vfsList);
         wasm.scopedAllocCall(()=>{
           const vfsArg = (v)=>wasm.xWrap.testConvertArg('sqlite3_vfs*',v);
           for(const v of vfsList){
@@ -2615,128 +2630,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
    }/*kvvfs sqlite3_js_vfs_create_file()*/)
   ;/* end kvvfs tests */
 
-  ////////////////////////////////////////////////////////////////////////
-  T.g('OPFS: Origin-Private File System',
-      (sqlite3)=>(sqlite3.opfs
-                  ? true : "requires Worker thread in a compatible browser"))
-    .t({
-      name: 'OPFS db sanity checks',
-      test: async function(sqlite3){
-        const filename = this.opfsDbFile = 'sqlite3-tester1.db';
-        const pVfs = this.opfsVfs = capi.sqlite3_vfs_find('opfs');
-        T.assert(pVfs);
-        const unlink = this.opfsUnlink =
-              (fn=filename)=>{wasm.sqlite3_wasm_vfs_unlink(pVfs,fn)};
-        unlink();
-        let db = new sqlite3.oo1.OpfsDb(filename);
-        try {
-          db.exec([
-            'create table p(a);',
-            'insert into p(a) values(1),(2),(3)'
-          ]);
-          T.assert(3 === db.selectValue('select count(*) from p'));
-          db.close();
-          db = new sqlite3.oo1.OpfsDb(filename);
-          db.exec('insert into p(a) values(4),(5),(6)');
-          T.assert(6 === db.selectValue('select count(*) from p'));
-          this.opfsDbExport = capi.sqlite3_js_db_export(db);
-          T.assert(this.opfsDbExport instanceof Uint8Array)
-            .assert(this.opfsDbExport.byteLength>0
-                    && 0===this.opfsDbExport.byteLength % 512);
-        }finally{
-          db.close();
-          unlink();
-        }
-      }
-    }/*OPFS db sanity checks*/)
-    .t({
-      name: 'OPFS export/import',
-      test: async function(sqlite3){
-        let db;
-        try {
-          const exp = this.opfsDbExport;
-          delete this.opfsDbExport;
-          capi.sqlite3_js_vfs_create_file("opfs", this.opfsDbFile, exp);
-          const db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
-          T.assert(6 === db.selectValue('select count(*) from p'));
-        }finally{
-          if(db) db.close();
-        }
-      }
-    }/*OPFS export/import*/)
-    .t({
-      name: 'OPFS utility APIs and sqlite3_js_vfs_create_file()',
-      test: async function(sqlite3){
-        const filename = this.opfsDbFile;
-        const pVfs = this.opfsVfs;
-        const unlink = this.opfsUnlink;
-        T.assert(filename && pVfs && !!unlink);
-        delete this.opfsDbFile;
-        delete this.opfsVfs;
-        delete this.opfsUnlink;
-        unlink();
-        // Sanity-test sqlite3_js_vfs_create_file()...
-        /**************************************************************
-           ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended
-           for client-side use. It is only for this project's own
-           internal use. Its APIs are subject to change or removal at
-           any time.
-        ***************************************************************/
-        const opfs = sqlite3.opfs;
-        const fSize = 1379;
-        let sh;
-        try{
-          T.assert(!(await opfs.entryExists(filename)));
-          capi.sqlite3_js_vfs_create_file(
-            pVfs, filename, null, fSize
-          );
-          T.assert(await opfs.entryExists(filename));
-          let fh = await opfs.rootDirectory.getFileHandle(filename);
-          sh = await fh.createSyncAccessHandle();
-          T.assert(fSize === await sh.getSize());
-          await sh.close();
-          sh = undefined;
-          unlink();
-          T.assert(!(await opfs.entryExists(filename)));
-
-          const ba = new Uint8Array([1,2,3,4,5]);
-          capi.sqlite3_js_vfs_create_file(
-            "opfs", filename, ba
-          );
-          T.assert(await opfs.entryExists(filename));
-          fh = await opfs.rootDirectory.getFileHandle(filename);
-          sh = await fh.createSyncAccessHandle();
-          T.assert(ba.byteLength === await sh.getSize());
-          await sh.close();
-          sh = undefined;
-          unlink();
-
-          T.mustThrowMatching(()=>{
-            capi.sqlite3_js_vfs_create_file(
-              "no-such-vfs", filename, ba
-            );
-          }, "SQLITE_NOTFOUND: Unknown sqlite3_vfs name: no-such-vfs");
-        }finally{
-          if(sh) await sh.close();
-          unlink();
-        }
-
-        // Some sanity checks of the opfs utility functions...
-        const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12);
-        const aDir = testDir+'/test/dir';
-        T.assert(await opfs.mkdir(aDir), "mkdir failed")
-          .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists")
-          .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)")
-          .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed")
-          .assert(!(await opfs.unlink(testDir+'/test/dir')),
-                  "delete 2b should have failed (dir already deleted)")
-          .assert((await opfs.unlink(testDir, true)), "delete 3 failed")
-          .assert(!(await opfs.entryExists(testDir)),
-                  "entryExists(",testDir,") should have failed");
-      }
-    }/*OPFS util sanity checks*/)
-  ;/* end OPFS tests */
-
   ////////////////////////////////////////////////////////////////////////
   T.g('Hook APIs')
     .t({
@@ -2942,8 +2835,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
               .assert( capi.sqlite3session_enable(pSession, -1) > 0 )
               .assert(undefined === db1.selectValue('select a from t where rowid=2'));
           }else{
-            warn("sqlite3session_enable() tests disabled due to unexpected results.",
-                 "(Possibly a tester misunderstanding, as opposed to a bug.)");
+            warn("sqlite3session_enable() tests are currently disabled.");
           }
           let db1Count = db1.selectValue("select count(*) from t");
           T.assert( db1Count === (testSessionEnable ? 2 : 3) );
@@ -3007,6 +2899,235 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
       }
     })/*session API sanity tests*/
   ;/*end of session API group*/;
+
+  ////////////////////////////////////////////////////////////////////////
+  T.g('OPFS: Origin-Private File System',
+      (sqlite3)=>(sqlite3.capi.sqlite3_vfs_find("opfs")
+                  || 'requires "opfs" VFS'))
+    .t({
+      name: 'OPFS db sanity checks',
+      test: async function(sqlite3){
+        const filename = this.opfsDbFile = 'sqlite3-tester1.db';
+        const pVfs = this.opfsVfs = capi.sqlite3_vfs_find('opfs');
+        T.assert(pVfs);
+        const unlink = this.opfsUnlink =
+              (fn=filename)=>{wasm.sqlite3_wasm_vfs_unlink(pVfs,fn)};
+        unlink();
+        let db = new sqlite3.oo1.OpfsDb(filename);
+        try {
+          db.exec([
+            'create table p(a);',
+            'insert into p(a) values(1),(2),(3)'
+          ]);
+          T.assert(3 === db.selectValue('select count(*) from p'));
+          db.close();
+          db = new sqlite3.oo1.OpfsDb(filename);
+          db.exec('insert into p(a) values(4),(5),(6)');
+          T.assert(6 === db.selectValue('select count(*) from p'));
+          this.opfsDbExport = capi.sqlite3_js_db_export(db);
+          T.assert(this.opfsDbExport instanceof Uint8Array)
+            .assert(this.opfsDbExport.byteLength>0
+                    && 0===this.opfsDbExport.byteLength % 512);
+        }finally{
+          db.close();
+          unlink();
+        }
+      }
+    }/*OPFS db sanity checks*/)
+    .t({
+      name: 'OPFS export/import',
+      test: async function(sqlite3){
+        let db;
+        try {
+          const exp = this.opfsDbExport;
+          delete this.opfsDbExport;
+          capi.sqlite3_js_vfs_create_file("opfs", this.opfsDbFile, exp);
+          const db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
+          T.assert(6 === db.selectValue('select count(*) from p'));
+        }finally{
+          if(db) db.close();
+        }
+      }
+    }/*OPFS export/import*/)
+    .t({
+      name: 'OPFS utility APIs and sqlite3_js_vfs_create_file()',
+      test: async function(sqlite3){
+        const filename = this.opfsDbFile;
+        const pVfs = this.opfsVfs;
+        const unlink = this.opfsUnlink;
+        T.assert(filename && pVfs && !!unlink);
+        delete this.opfsDbFile;
+        delete this.opfsVfs;
+        delete this.opfsUnlink;
+        unlink();
+        // Sanity-test sqlite3_js_vfs_create_file()...
+        /**************************************************************
+           ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended
+           for client-side use. It is only for this project's own
+           internal use. Its APIs are subject to change or removal at
+           any time.
+        ***************************************************************/
+        const opfs = sqlite3.opfs;
+        const fSize = 1379;
+        let sh;
+        try{
+          T.assert(!(await opfs.entryExists(filename)));
+          capi.sqlite3_js_vfs_create_file(
+            pVfs, filename, null, fSize
+          );
+          T.assert(await opfs.entryExists(filename));
+          let fh = await opfs.rootDirectory.getFileHandle(filename);
+          sh = await fh.createSyncAccessHandle();
+          T.assert(fSize === await sh.getSize());
+          await sh.close();
+          sh = undefined;
+          unlink();
+          T.assert(!(await opfs.entryExists(filename)));
+
+          const ba = new Uint8Array([1,2,3,4,5]);
+          capi.sqlite3_js_vfs_create_file(
+            "opfs", filename, ba
+          );
+          T.assert(await opfs.entryExists(filename));
+          fh = await opfs.rootDirectory.getFileHandle(filename);
+          sh = await fh.createSyncAccessHandle();
+          T.assert(ba.byteLength === await sh.getSize());
+          await sh.close();
+          sh = undefined;
+          unlink();
+
+          T.mustThrowMatching(()=>{
+            capi.sqlite3_js_vfs_create_file(
+              "no-such-vfs", filename, ba
+            );
+          }, "SQLITE_NOTFOUND: Unknown sqlite3_vfs name: no-such-vfs");
+        }finally{
+          if(sh) await sh.close();
+          unlink();
+        }
+
+        // Some sanity checks of the opfs utility functions...
+        const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12);
+        const aDir = testDir+'/test/dir';
+        T.assert(await opfs.mkdir(aDir), "mkdir failed")
+          .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists")
+          .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)")
+          .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed")
+          .assert(!(await opfs.unlink(testDir+'/test/dir')),
+                  "delete 2b should have failed (dir already deleted)")
+          .assert((await opfs.unlink(testDir, true)), "delete 3 failed")
+          .assert(!(await opfs.entryExists(testDir)),
+                  "entryExists(",testDir,") should have failed");
+      }
+    }/*OPFS util sanity checks*/)
+  ;/* end OPFS tests */
+
+  ////////////////////////////////////////////////////////////////////////
+  T.g('OPFS SyncAccessHandle Pool VFS',
+      (sqlite3)=>(hasOpfs() || "requires OPFS APIs"))
+    .t({
+      name: 'SAH sanity checks',
+      test: async function(sqlite3){
+        T.assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name))
+          .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) < 0)
+        const inst = sqlite3.installOpfsSAHPoolVfs,
+              catcher = (e)=>{
+                error("Cannot load SAH pool VFS.",
+                      "This might not be a problem,",
+                      "depending on the environment.");
+                return false;
+              };
+        let u1, u2;
+        // Ensure that two immediately-consecutive installations
+        // resolve to the same Promise instead of triggering
+        // a locking error.
+        const P1 = inst(sahPoolConfig).then(u=>u1 = u).catch(catcher),
+              P2 = inst(sahPoolConfig).then(u=>u2 = u).catch(catcher);
+        await Promise.all([P1, P2]);
+        if(!(await P1)) return;
+        T.assert(u1 === u2)
+          .assert(sahPoolConfig.name === u1.vfsName)
+          .assert(sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name))
+          .assert(u1.getCapacity() >= sahPoolConfig.initialCapacity
+                  /* If a test fails before we get to nuke the VFS, we
+                     can have more than the initial capacity on the next
+                     run. */)
+          .assert(u1.getCapacity() + 2 === (await u2.addCapacity(2)))
+          .assert(2 === (await u2.reduceCapacity(2)))
+          .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) >= 0);
+
+        T.assert(0 === u1.getFileCount());
+        const dbName = '/foo.db';
+        let db = new u1.OpfsSAHPoolDb(dbName);
+        T.assert(db instanceof sqlite3.oo1.DB)
+          .assert(1 === u1.getFileCount());
+        db.exec([
+          'create table t(a);',
+          'insert into t(a) values(1),(2),(3)'
+        ]);
+        T.assert(1 === u1.getFileCount());
+        T.assert(3 === db.selectValue('select count(*) from t'));
+        db.close();
+        T.assert(1 === u1.getFileCount());
+        db = new u2.OpfsSAHPoolDb(dbName);
+        T.assert(1 === u1.getFileCount());
+        db.close();
+        const fileNames = u1.getFileNames();
+        T.assert(1 === fileNames.length)
+          .assert(dbName === fileNames[0])
+          .assert(1 === u1.getFileCount())
+
+        if(1){ // test exportFile() and importDb()
+          const dbytes = u1.exportFile(dbName);
+          T.assert(dbytes.length >= 4096);
+          const dbName2 = '/exported.db';
+          u1.importDb(dbName2, dbytes);
+          T.assert( 2 == u1.getFileCount() );
+          let db2 = new u1.OpfsSAHPoolDb(dbName2);
+          T.assert(db2 instanceof sqlite3.oo1.DB)
+            .assert(3 === db2.selectValue('select count(*) from t'));
+          db2.close();
+          T.assert(true === u1.unlink(dbName2))
+            .assert(false === u1.unlink(dbName2))
+            .assert(1 === u1.getFileCount())
+            .assert(1 === u1.getFileNames().length);
+        }
+
+        T.assert(true === u1.unlink(dbName))
+          .assert(false === u1.unlink(dbName))
+          .assert(0 === u1.getFileCount())
+          .assert(0 === u1.getFileNames().length);
+
+        // Demonstrate that two SAH pools can coexist so long as
+        // they have different names.
+        const conf2 = JSON.parse(JSON.stringify(sahPoolConfig));
+        conf2.name += '-test2';
+        const POther = await inst(conf2);
+        log("Installed second SAH instance as",conf2.name);
+        T.assert(0 === POther.getFileCount())
+          .assert(true === await POther.removeVfs());
+
+        if(0){
+           /* Enable this block to inspect vfs's contents via the dev
+              console or OPFS Explorer browser extension.  The
+              following bits will remove them. */
+          return;
+        }
+        T.assert(true === await u2.removeVfs())
+          .assert(false === await u1.removeVfs())
+          .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name));
+
+        let cErr, u3;
+        conf2.$testThrowInInit = new Error("Testing throwing during init.");
+        conf2.name = sahPoolConfig.name+'-err';
+        const P3 = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e);
+        T.assert(P3 === conf2.$testThrowInInit)
+          .assert(cErr === P3)
+          .assert(undefined === u3)
+          .assert(!sqlite3.capi.sqlite3_vfs_find(conf2.name));
+      }
+    }/*OPFS SAH Pool sanity checks*/)
+
   ////////////////////////////////////////////////////////////////////////
   T.g('Bug Reports')
     .t({
@@ -3088,11 +3209,15 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
   globalThis.sqlite3InitModule({
     print: log,
     printErr: error
-  }).then(function(sqlite3){
-    //console.log('sqlite3 =',sqlite3);
+  }).then(async function(sqlite3){
     log("Done initializing WASM/JS bits. Running tests...");
     sqlite3.config.warn("Installing sqlite3 bits as global S for local dev/test purposes.");
     globalThis.S = sqlite3;
+    /*await sqlite3.installOpfsSAHPoolVfs(sahPoolConfig)
+      .then((u)=>log("Loaded",u.vfsName,"VFS"))
+      .catch(e=>{
+        log("Cannot install OpfsSAHPool.",e);
+      });*/
     capi = sqlite3.capi;
     wasm = sqlite3.wasm;
     log("sqlite3 version:",capi.sqlite3_libversion(),
@@ -3107,6 +3232,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
     }else{
       logClass('warning',"sqlite3_wasm_test_...() APIs unavailable.");
     }
+    log("registered vfs list =",capi.sqlite3_js_vfs_list().join(', '));
     TestUtil.runTests(sqlite3);
   });
 })(self);

main.mk

@@ -344,6 +344,7 @@ TESTSRC += \
   $(TOP)/ext/misc/amatch.c \
   $(TOP)/ext/misc/appendvfs.c \
   $(TOP)/ext/misc/basexx.c \
+  $(TOP)/ext/misc/bgckpt.c \
   $(TOP)/ext/misc/carray.c \
   $(TOP)/ext/misc/cksumvfs.c \
   $(TOP)/ext/misc/closure.c \

manifest

@@ -1,11 +1,11 @@
-C Mark\san\sunreachable\sbranch\sas\sNEVER().
-D 2023-07-21T22:22:13.828
+C Add\stests\sfor\sstructured-exception-handling\sand\swal2.\sFix\sa\sproblem\swith\sthe\ssame.
+D 2023-08-10T21:20:34.013
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
-F Makefile.in 0f4cb3955aaff8a40ec3857ba1784bd98b69802e51eff979f874b65713b627b2
-F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241
-F Makefile.msc 7248d860f71ab164b4cec3c415e6cc1bd9fee860c370d65bd8bb49e9572521e2
+F Makefile.in 3a780c12cb464f43e680a4f36f59615d45acf5a6762b56c205b6ef51ebd26c6d
+F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6
+F Makefile.msc 0b57ab2867b1fdc90c4e35a6777a3b24b81006cc7ba947c8db13929b4edd3800
 F README.md c1c4218efcc4071a6e26db2b517fdbc1035696a29b370edd655faddbef02b224
 F VERSION c6366dc72582d3144ce87b013cc35fe48d62f6d07d5be0c9716ea33c862144aa
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -15,7 +15,7 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am a8d1d24affe52ebf8d7ddcf91aa973fa0316618ab95bb68c87cabf8faf527dc8
 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac
-F autoconf/Makefile.msc 20366d19fbfc3fceecce95344a2114069eaab4fc22e4f02430da167a1e2ddf04
+F autoconf/Makefile.msc 00f11ce1f7904416fe841c33e7d789defe8c39e1df6b97e93ed2af3b1bbaf9d7
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 42cfd21d0b19dc7d5d85fb5c405c5f3c6a4c923021c39128f6ba685355d8fd56
 F autoconf/configure.ac ec7fa914c5e74ff212fe879f9bb6918e1234497e05facfb641f30c4d5893b277
@@ -44,6 +44,7 @@ F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d
 F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56
 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a
 F doc/wal-lock.md 781726aaba20bafeceb7ba9f91d5c98c6731691b30c954e37cf0b49a053d461d
+F doc/wal2.md a807405a05e19a4945c5905a9ffa0fe45b8560dd7572461192501f565c19cdb5
 F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94
@@ -85,16 +86,16 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
-F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h ed48a096418ff4a7c02ac9bd1e8d40c46de21b79a132b8b08d3f32233703de7d
+F ext/fts5/fts5.h 9bebc9fb8b75b0777e741c758540e07c3c80ce75204198979028fe6cc4c59486
+F ext/fts5/fts5Int.h 78a63cc0795186cde5384816a9403a68c65774b35d952e05b81a1b4b158e07c8
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 051056a9052f5d3a4d1c695f996fd364f920e341f136c60ab2c04aa7e267113f
-F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
-F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 430d3edfc5b51d02f0671e2b8268c12bf485f7dc80db2b61c996e74814bc46b0
-F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d
-F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082
+F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
+F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
+F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
+F ext/fts5/fts5_index.c 93b4cd116b76b6adf224cd3d213f1e06cfe10ae0eb21d6372b1c53b8f0c382a3
+F ext/fts5/fts5_main.c 7070031993ba5b5d89b13206ec4ef624895f2f7c0ec72725913d301e4d382445
+F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -132,6 +133,11 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
+F ext/fts5/test/fts5contentless.test eb31159d62811b3a32fb1cfb36be20f9d9db75637c7fe6aa7f5c533db2d00576
+F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
+F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286
+F ext/fts5/test/fts5contentless4.test 0f43ededc2874f65d7da99b641a82239854d98d3fa43db729f284b723f23b69f
+F ext/fts5/test/fts5contentless5.test 216962d51376b62e4ff45e8db00aa3e1ab548cb36b7fd450633e73b2d678f88e
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -159,6 +165,7 @@ F ext/fts5/test/fts5faultA.test be4487576bff8c22cee6597d1893b312f306504a8c6ccd3c
 F ext/fts5/test/fts5faultB.test d606bdb8e81aaeb6f41de3fc9fc7ae315733f0903fbff05cf54f5b045b729ab5
 F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860bbf9e504b9647996
 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075
+F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079
 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
@@ -277,6 +284,7 @@ F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e
 F ext/misc/base64.c a71b131e50300c654a66c469a25b62874481f3d1cb3beb56aca9a68edd812e0d
 F ext/misc/base85.c 073054111988db593ef5fdb87ab8c459df1ea0c3aaaddf0f5bfa3d72b7e6280a
 F ext/misc/basexx.c 89ad6b76558efbceb627afd5e2ef1d84b2e96d9aaf9b7ecb20e3d00b51be6fcf
+F ext/misc/bgckpt.c 49ae19aa03e6da393db5d17da256374d4c4d36889fdd89d6e4bc93aca2b752e6
 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a
 F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9
 F ext/misc/carray.c 0ba03f1e6647785d4e05b51be567f5652f06941314ff9d3d3763900aa353b6b5
@@ -475,62 +483,63 @@ F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a
 F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795
 F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c
 F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc
-F ext/session/sqlite3session.c e50a9218ee360db0a25298adc6614162d80ebe65d3f6a5b0a021e0902f6536a1
+F ext/session/sqlite3session.c 1971b61ca45babf0d9e4bb669a65b0903135e9828af2fcd4f0c8f1b7acf36b6f
 F ext/session/sqlite3session.h 653e9d49c4edae231df8a4c8d69c2145195aedb32462d4b44229dbee7d2680fb
 F ext/session/test_session.c 5285482f83cd92b4c1fe12fcf88210566a18312f4f2aa110f6399dae46aeccbb
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
-F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
-F ext/wasm/GNUmakefile 74e351ff45b4061cfed8df237d301819a04182ae304a99118883b064baa25fc2
+F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3
+F ext/wasm/GNUmakefile ddf1aede4275e404c7eda782462c33b6406fcd2dd327241f6b22c0f7b80938e4
 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576
-F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9
-F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab
+F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193
+F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33
 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b
 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287
-F ext/wasm/api/README.md 77a2f1f2fc60a35def7455dffc8d3f2c56385d6ac5c6cecc60fa938252ea2c54
-F ext/wasm/api/extern-post-js.c-pp.js 116749b7e55b7519129de06d3d353e19df68cfb24b12204aa4dc30c9a83023fe
+F ext/wasm/api/README.md 5eb44fa02e9c693a1884a3692428647894b0380b24bca120866b7a24c8786134
+F ext/wasm/api/extern-post-js.c-pp.js c4154a7f90c2d7e51fd6738273908152036c3457fdc0b6523f1be3ef51105aac
 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41
 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1
 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62
 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219
-F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf
-F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803
+F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e
+F ext/wasm/api/sqlite3-api-glue.js cc6b0bb093bdb6279d4af259200b7b9e150e3796a8a3a4cd09a4928c43d25e56
 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8
-F ext/wasm/api/sqlite3-api-prologue.js 7d1c1ef59b9dcc42ad3a9cec9da972c42e29316a270cd126e7f660509b09027b
+F ext/wasm/api/sqlite3-api-prologue.js 76258e160bf6a89cc75a7d3c05646a054c8cab7219cd1e10bc20cacaad022131
 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec
 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
-F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2
-F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487
-F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87
-F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda
-F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f
+F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379
+F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25
+F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js abb69b5e008961026bf5ff433d7116cb046359af92a5daf73208af2e7ac80ae7
+F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e7a690e0e78ff4d563f2eca468f91db69f001ff4b79c6d2304cbb6f62dca437d
+F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f
+F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js f10c3ecd9df06f6320073c2ce230a7ed7c56034d8b88c1e57095f2a97faf423a
 F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75
 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8
 F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c27df8fdac93e
 F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4
 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51
 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15
-F ext/wasm/common/testing.css 0ff15602a3ab2bad8aef2c3bd120c7ee3fd1c2054ad2ace7e214187ae68d926f
-F ext/wasm/common/whwasmutil.js ae263dec9d7384f4c530f324b99d00516a4d6f26424372daee65031e00eb49b3
+F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f
+F ext/wasm/common/whwasmutil.js 4c64594eecc7af4ae64259e95a71ba2a7edf118881aaff0bba86d0c7164e78e4
 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed
 F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508
-F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6
+F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5
 F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e
 F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8
 F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f
 F ext/wasm/demo-worker1-promiser.js 5e5c7d7c91cd7aae9cc733afd02569ba9c6928292db413b550e8b842f4b75e87
 F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d
 F ext/wasm/demo-worker1.js 836bece8615b17b1b572584f7b15912236a5947fe8c68b98d2737d7e287447ef
-F ext/wasm/dist.make 451fb1b732257849f6e898d2a862512a0401500ed369ef53bdfeddf9c77bc3b9
+F ext/wasm/dist.make 3a851858aad72e246a5d9c5aaf6b6a144305f1bf898ac1846760ea7bab95c9a3
 F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f
-F ext/wasm/fiddle.make dbe36b90b8907ae28ecb9c0e9fd8389dbdaecf117ea4fb2ea33864bdfa498a94
+F ext/wasm/fiddle.make fa2ba6e90457ba2a71cb745f200d409caf773076df75ae5b177cc225d7627a11
 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f
-F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa061ce8776a6e75da8a10
+F ext/wasm/fiddle/fiddle-worker.js e0153f9af6500805c6f09c0b3cfdb7d857e9d6863dbee9d50d1628fccf5f4b4d
+F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08
 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715
-F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2
 F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e
-F ext/wasm/index.html b768e8659b4fe311912e54d42906449d51c0f84b7f036cca47ec1f93bf3f91de
+F ext/wasm/index.html 4e7847b909f4ae0da8c829b150b79454050e53b3658431f138636257729cd42b
 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54
 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb
 F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10
@@ -538,8 +547,8 @@ F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d
 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63
 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d
 F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe
-F ext/wasm/speedtest1-worker.html 97c2bf5f8534091ce718de05801090d5a80c3f13575996f095ba23638e1bdca0
-F ext/wasm/speedtest1-worker.js 13b57c4a41729678a1194014afec2bd5b94435dcfc8d1039dfa9a533ac819ee1
+F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5
+F ext/wasm/speedtest1-worker.js 315d26198c46be7c85e26fda15d80ef882424276abde25ffd8b026fb02a35d8c
 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
@@ -548,7 +557,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555
 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b
 F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c
 F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2
-F ext/wasm/tester1.c-pp.js 4420eb97b6b4fc79e4e156b4b8010dd9f373365f4230dd76d823fb04ce28ffde
+F ext/wasm/tester1.c-pp.js b88dcad5424a652e8204c44a71bbc3deb22a4922c97ba792aedbabb7a6827b91
 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1
 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d
 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
@@ -557,7 +566,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0
-F main.mk 0eb735008653412026092961cecdf7d698156c060e4062a69d911335982d471c
+F main.mk b7cf0350144fdf7551ecca064272b88098dcf206cb0d4c79754ad734a8f1c24b
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421
@@ -576,31 +585,31 @@ F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4
 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523
 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645
 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522
-F src/btree.c 2281facb0531d53fb42c03d1f32bc1b5903564d782ec5ff4ffc63171d960e2aa
-F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc
-F src/btreeInt.h 3b4eff7155c0cea6971dc51f62e3529934a15a6640ec607dd42a767e379cb3a9
+F src/btree.c a2f42c88fd30300df4c5d8b04a366b78844355b76a74acf46b6e68d36f33a7e0
+F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240
+F src/btreeInt.h 91a9e0c41a0e71fa91a742ec285c63dd8dcb38b73d14fae0ed7209174ff0fdc1
 F src/build.c a8ae3b32d9aa9bbd2c0e97d7c0dd80def9fbca408425de1608f57ee6f47f45f4
 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
-F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d
+F src/ctime.c f439c21d439aaf4706950a2597474f1a796b557a0750666308202c0f601ef0fd
 F src/date.c f73f203b3877cef866c60ab402aec2bf89597219b60635cf50cbe3c5e4533e94
 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387
 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef
-F src/delete.c cd5f5cd06ed0b6a882ec1a8c2a0d73b3cecb28479ad19e9931c4706c5e2182be
-F src/expr.c ef4a81822da6f767696bd7f4b9983328af061158958138540142285a5b1181b7
+F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500
+F src/expr.c 1affe0cc049683ef0ef3545d9b6901508556b0ef7e2892a344c3df6d7288d79d
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c a7fcbf7e66d14dbb73cf49f31489ebf66d0e6006c62b95246924a3bae9f37b36
-F src/func.c 25f2e0204c011be56fc3c9a180534b68ca4866c61ec19806880136450434112d
+F src/func.c cc1da67fd643a43cfe691784158ec656d8ec6d13bb17e67018b01b38b3e4f5ab
 F src/global.c 29f56a330ed9d1b5cd9b79ac0ca36f97ac3afc730ff8bfa987b0db9e559d684d
 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 46ea5566e1363f4f353db79b0378c2bf8ffdf9d4667daee3df67b14669767bed
+F src/json.c 9c231a853268ce6aee2e300e26d4445ba42117374a2275f8e9537b2f912909d6
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
-F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
+F src/main.c fde8f13c876a658b4e8b74b77d875ca887915c174ea6a2f3122d80966f93d865
 F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2
@@ -621,33 +630,33 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06
 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a
 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107
-F src/os_unix.c 95b407307deb902a3bd9a5d5666c7838709cccb337baeee6ef0a53f512d3673e
+F src/os_unix.c 2e8b12107f75d1bd16412f312b4c5d5103191807a37836d3b81beb26436ad81b
 F src/os_win.c 7038223a1cda0a47e2ab4db47f63bf1833fe53ba0542f0f283a062ea13894103
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 5ddf3a74c633a008ea6b2f5b3186167e88e2c8ca8a252ecab06ab3f1eb48e60f
-F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3
-F src/parse.y 8828f9e15f04d469eab9c0f2aed504e534b1c97c68836bed6f07afab29c2ac0b
+F src/pager.c e7af170f5799a5edd77fc267e81cb0292e1a1cb7b488b62c6e2fff78aedc2efb
+F src/pager.h ca3b415a3f03359d5692b71ec0ba068a58ce447fac7312f89293280d95ee38da
+F src/parse.y aeb7760d41cfa86465e3adba506500c021597049fd55f82a30e5b7045862c28c
 F src/pcache.c 4cd4a0043167da9ba7e19b4d179a0e6354e7fe32c16f781ecf9bf0a5ff63b40b
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
-F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a
+F src/pragma.c 3a7c0f8e06b94ee34ded7468002b4a502fcdd23bb58c3677a185a4addae7604b
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
-F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1
-F src/printf.c 84b7b4b647f336934a5ab2e7f0c52555833cc0778d2d60e016cca52ee8c6cd8f
+F src/prepare.c 80548297dc0e1fb3139cdebffb5a1bcac3dfac66d791012dd74838e70445072d
+F src/printf.c e3ba080e2f409f9bfcc8d34724e6fc160e9c718dc92d0548f6b71b8b6f860ce2
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
-F src/select.c 3ab1186290a311a8ceed1286c0e286209f7fe97b2d02c7593258004ce295dd88
-F src/shell.c.in d320d8a13636de06d777cc1eab981caca304e175464e98183cf4ea68d93db818
-F src/sqlite.h.in f999ef3642f381d69679b2516b430dbcb6c5a2a951b7f5e43dc4751b474a5774
+F src/select.c 5f545a2c8702d4d3430bbb188cfec47d6c122d899061ef00cbe56af14591c574
+F src/shell.c.in 694aaf751f00610381533d4a31c83d142cfc83ef91ef65e2aa6912ace7c39b40
+F src/sqlite.h.in 73a366c1c45d5ac9888cfe81c458826a44498531d106cfb4f328193ab5f6f17d
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
-F src/sqliteInt.h dcb1a885e8b6cb78df618944b89d44361a99d0fe33e1bba2c150a855f7dc5599
+F src/sqliteInt.h 025ed58a41968ef80d64cdc194caa8dd207b0256b147253d762fdac7a62408f9
 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
 F src/tclsqlite.c ecbc3c99c0d0c3ed122a913f143026c26d38d57f33e06bb71185dd5c1efe37cd
-F src/test1.c 9111b12427b3b94429aff68bbcf5b125ae70438f6ce2f3f033e2a69626ec26c9
+F src/test1.c ebba2473874a23add4a10881b90bd445cfa7f59f90749434938aec14239c6486
 F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef
 F src/test3.c e5178558c41ff53236ae0271e9acb3d6885a94981d2eb939536ee6474598840e
 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664
@@ -667,7 +676,7 @@ F src/test_demovfs.c 38a459d1c78fd9afa770445b224c485e079018d6ac07332ff9bd07b54d2
 F src/test_devsym.c 649434ed34d0b03fbd5a6b42df80f0f9a7e53f94dd1710aad5dd8831e91c4e86
 F src/test_fs.c ba1e1dc18fd3159fdba0b9c4256f14032159785320dfbd6776eb9973cb75d480
 F src/test_func.c 24df3a346c012b1fc9e1001d346db6054deb426db0a7437e92490630e71c9b0a
-F src/test_hexio.c 9478e56a0f08e07841a014a93b20e4ba2709ab56d039d1ca8020e26846aa19bd
+F src/test_hexio.c a605a100e628d39330044ae5f34cb19d50843061ed3178c3f83b37aef65f7e0a
 F src/test_init.c f2cc4774b7c9140f76e45ecbb2ae219f68e3acbbe248c0179db666a70eae9f08
 F src/test_intarray.c 39b4181662a0f33a427748d87218e7578d913e683dc27eab7098bb41617cac71
 F src/test_intarray.h 6c3534641108cd1bea517a8e117dcba237081310a29a4c35bd2190caa8972293
@@ -688,7 +697,7 @@ F src/test_schema.c f5d6067dfc2f2845c4dd56df63e66ee826fb23877855c785f75cc2ca83fd
 F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a
 F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e
 F src/test_syscall.c 9fdb13b1df05e639808d44fcb8f6064aaded32b6565c00b215cfd05a060d1aca
-F src/test_tclsh.c 3ff5d188a72f00807425954ea3b493dfd3a4b890ecc6700ea83bad2fd1332ecf
+F src/test_tclsh.c 01c76e2ee362cda246710c476351b105371cf4620857c8492121858744998dc9
 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc
 F src/test_thread.c 7ddcf0c8b79fa3c1d172f82f322302c963d923cdb503c6171f3c8081586d0b01
 F src/test_vdbecov.c f60c6f135ec42c0de013a1d5136777aa328a776d33277f92abac648930453d43
@@ -702,27 +711,27 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 0fb405f9adf3f757c26bfc1ae6d58ac5dccbb918917ba9e5ef0e6673a06563d3
 F src/treeview.c 1d52fbc4e97161e65858d36e3424ea6e3fc045dd8a679c82b4b9593dc30de3bd
 F src/trigger.c ad6ab9452715fa9a8075442e15196022275b414b9141b566af8cdb7a1605f2b0
-F src/update.c 0aa36561167a7c40d01163238c297297962f31a15a8d742216b3c37cdf25f731
+F src/update.c 0bb9171afaa4d0b100ad946873bccda7aef90ffe083ef5c63668fce08c4df9da
 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
-F src/util.c c2aa170f2eb429235b1dddce8952770787ffa5124dc89d405bfbe8ebad8e7ebd
+F src/util.c a40062117e705eb3339201842717a022092816b92479eead6397cde28af32ff9
 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
-F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12
+F src/vdbe.c 11f1a00443991bd12bd0c97050b2614f7fd3e82b9cc0ffdb13d14d424d733e77
 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0
-F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7
-F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7
-F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339
+F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c
+F src/vdbeapi.c f37822f215740ede2a8fcae99bc13f2cc3a72dd0e1d22b81b9298c5ca67dbc38
+F src/vdbeaux.c e3aa5c46827cd95e0fc4d0f302fa3e901ab5f07258fdbb42709eeef40f63018d
 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce
-F src/vdbemem.c b3013dd11696db423bb410a50126d46d700fe3133b936a32195700e9731d2960
+F src/vdbemem.c 317b9f48708139db6239ade40c7980b4bc8233168383690d588dad6d8437f722
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
-F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac
+F src/vdbevtab.c 57fa8f56478e5b5cb558cb425e7878515e0a105c54f96f1d1bbf4b9433529254
 F src/vtab.c 1ecf8c3745d29275688d583e12822fa984d421e0286b5ef50c137bc3bf6d7a64
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c dd843f619ac60d5dadab7109cf402432ba74dde0c301505fd1c202add07659e3
-F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
+F src/wal.c 1ea71da11648befe8be082d0ef6adad144435a827d7ae1593da1bf375482c21d
+F src/wal.h a3e64f463acf027f041c2081841e15e8c81c56d0daa45e308df423d9d66fd5a7
 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2
-F src/where.c 477fcc5e561ef169e6002499602af6b805156c2aae6b2f5c2c93ef8c1cd64768
+F src/where.c b8917792f1e0dbfa28fb29e6cd3d560060d69667be0ba4c491cbc772363264f5
 F src/whereInt.h c7d19902863beadec1d04e66aca39c0bcd60b74f05f0eaa7422c7005dfc5d51a
 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1
 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00
@@ -892,7 +901,7 @@ F test/corrupt6.test fc6a891716139665dae0073b6945e3670bf92568
 F test/corrupt7.test ffa86896fe63a3d00b0a131e1e64f402e4da9f7e5d89609d6501c851e511d73a
 F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516
 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85
-F test/corruptA.test 112f4b2ae0b95ebf3ea63718642fb969a93acea557ace3a307234d19c245989b
+F test/corruptA.test 56e8f321adaf3411960e9d2c7136669d8e1a91cbde6cf401ea84e6d6c7ccbe10
 F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec
 F test/corruptC.test 9cf32275dae3ca33f645afe5d1d3f5ba5ac2af2b0833dfb5282f9dccb6fb81bb
 F test/corruptD.test a828c788535946a372a56a750b242cd96287cd823657abe5a73c5e51b91bdd28
@@ -903,7 +912,7 @@ F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454
 F test/corruptI.test 9d8cbf6214e492abe9e822e759b9751ae336cec0a6fe3ff3b37bfbd8ff9c22ca
 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4
 F test/corruptK.test 5b4212fe346699831c5ad559a62c54e11c0611bdde1ea8423a091f9c01aa32af
-F test/corruptL.test b42978028afc5eefc8b51d8d7cd6a9344ba7362d7ed4511ee2070f56e06d5a1c
+F test/corruptL.test 504d90502d9993440226edc355d2275524b89064ea3df5ee5c27f7028ec59d07
 F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067
 F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4
 F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576
@@ -1130,9 +1139,9 @@ F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5b
 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
 F test/fuzzdata4.db b502c7d5498261715812dd8b3c2005bad08b3a26e6489414bd13926cd3e42ed2
 F test/fuzzdata5.db e35f64af17ec48926481cfaf3b3855e436bd40d1cfe2d59a9474cb4b748a52a5
-F test/fuzzdata6.db 92a80e4afc172c24f662a10a612d188fb272de4a9bd19e017927c95f737de6d7
+F test/fuzzdata6.db b8725a5f5cf7a3b7241a9038e57ca7e7cc8c3f4d86b44bd770617bda245ab2b0
 F test/fuzzdata7.db 0166b56fd7a6b9636a1d60ef0a060f86ddaecf99400a666bb6e5bbd7199ad1f2
-F test/fuzzdata8.db f6c2f2af4deaaae0ddb3310d509c2659990794aa653dc501b80a0534c3493f80
+F test/fuzzdata8.db 40c85daae47da64387c3dab7bbd99c21e425c0bfdb4b149cb685b1ab474a2cb4
 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8
 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14
 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc
@@ -1222,12 +1231,12 @@ F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ff
 F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946
 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
-F test/json/README.md 0992b8ccbecd8424b84c1173f9ac56bcc0ae96d49912fd2d86c04eb1f3cba7af
+F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2
 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f
-F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d
+F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307
 F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x
 F test/json101.test 94126d4291d4a00e45f6988ce885c410de69243490e46e70e9946cb6e6f9ea02
-F test/json102.test 13dc9e7b7f359ecb861e02f9bd7019f7342a63d1c354273b0a8f3904050560a8
+F test/json102.test 24f6f204f9cde45b971016691d0b92a9b4c58040d699e36d6b12cb165f9083ff
 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe
 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1
 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d
@@ -1364,7 +1373,7 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/pendingrace.test cbdf0f74bc939fb43cebad64dda7a0b5a3941a10b7e9cc2b596ff3e423a18156
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 8bd6b6db541e2a7f9bb894be99ef5c00526b23762c4a00c574e1cba697495125
+F test/permutations.test 1f9b143250923886db3d5793e703a51b892e7f3fcd0e8cff5b6e30915687afe0
 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f
 F test/pragma.test 57a36226218c03cfb381019fe43234b2cefbd8a1f12825514f906a17ccf7991e
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
@@ -1388,7 +1397,7 @@ F test/quote.test ffb40f0eb7a25c1d8cfe11ee2fe67f8e85fbf3fed348810834114be1fdada1
 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459
 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df
 F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736
-F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8
+F test/rdonly.test 21e99ee237265d0cf95a0c84b50c784e834acaa4ef05d92a27b262626a656682
 F test/recover.test fd5199f928757cb308661b5fdca1abc19398a798ff7f24b57c3071e9f8e0471e
 F test/regexp1.test 8f2a8bc1569666e29a4cee6c1a666cd224eb6d50e2470d1dc1df995170f3e0f1
 F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5
@@ -1419,11 +1428,11 @@ F test/rowvaluefault.test 963ae9cdaed30a85a29668dd514e639f3556cae903ee9f172ea972
 F test/rowvaluevtab.test cd9747bb3f308086944c07968f547ad6b05022e698d80b9ffbdfe09ce0b8da6f
 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798
 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09
-F test/savepoint.test 6e9804a17767f08432c7a5e738b9a8f4b891d243110b63d3a41d270d3d1378ec
+F test/savepoint.test 63a120ec4fbbd5025b238c259d12ed0516fbf4bca6384041cb995ade9a5f00d2
 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7
 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0
 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd
-F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7
+F test/savepoint6.test 48a645a7bb3a59a6fcf06a7364cfe5b655c336760de39068f7c241b0fc80d963
 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa
 F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2
 F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e
@@ -1492,7 +1501,7 @@ F test/snapshot.test a504f2e7009f512ef66c719f0ea1c55a556bdaf1e1312c80a04d46fc1a3
 F test/snapshot2.test 8d6ff5dd9cc503f6e12d408a30409c3f9c653507b24408d9cd7195931c89bc54
 F test/snapshot3.test 8744313270c55f6e18574283553d3c5c5fe4c5970585663613a0e75c151e599b
 F test/snapshot4.test d4e9347ef2fcabc491fc893506c7bbaf334da3be111d6eb4f3a97cc623b78322
-F test/snapshot_fault.test f6c5ef7cb93bf92fbb4e864ecc5c87df7d3a250064838822db5b4d3a5563ede4
+F test/snapshot_fault.test 129234ceb9b26a0e1000e8563a16e790f5c1412354e70749cbd78c3d5d07d60a
 F test/snapshot_up.test a0a29c4cf33475fcef07c3f8e64af795e24ab91b4cc68295863402a393cdd41c
 F test/soak.test 18944cf21b94a7fe0df02016a6ee1e9632bc4e8d095a0cb49d95e15d5cca2d5c
 F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087
@@ -1555,7 +1564,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede
+F test/tester.tcl e6b9adcf4e56aa5c658501c10650646fee467b85d51a4ac80263d2cdae3993ba
 F test/testrunner.tcl 59490f189cac99b16b0376d0cc0a7ecfb753a84b89c9f4c361af337d88db53ac
 F test/testrunner_data.tcl 8169c68654ac8906833b8a6aadca973358a441ebf88270dd05c153e5f96f76b8
 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899
@@ -1764,7 +1773,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264
 F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2
 F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97
 F test/unordered.test 0edaf3411d300693bca595897c5201421c6c5ec787990a1dfe2f7f60ae93f1e2
-F test/update.test 90772ede84cfc779fc3d31884a84ec4c74deb501eeb09a1c6c91c03d8e94c0d8
+F test/update.test 85d3f46d0863033370bd881b1097f5694369a8730e53c5f85d96f32b7b310b47
 F test/update2.test 67455bc61fcbcf96923c45b3bc4f87bc72be7d67575ad35f134906148c7b06d3
 F test/upfrom1.tcl 8859d9d437f03b44174c4524a7a734a391fd4526fcff65be08285dafc9dc9041
 F test/upfrom1.test 8cb06689e99cd707d884faa16da0e8eb26ff658bb01c47ddf72fadade666e6e1
@@ -1778,7 +1787,7 @@ F test/upsert3.test 88d7d590a1948a9cb6eac1b54b0642f67a9f35a1fc0f19b200e97d5d39e3
 F test/upsert4.test 25d2a1da92f149331ae0c51ca6e3eee78189577585eab92de149900d62994fa5
 F test/upsert5.test fff0dcfce73c649204543088d8e5bde01172676063ec9b8f8fc7f195abc386fe
 F test/upsertfault.test f21ca47740841fdb4d61acfa7b17646d773e67724fe8c185b71c018db8a94b35
-F test/uri.test c1abaaaa28e9422d61e5f3f9cbc8ef993ec49fe802f581520731708561d49384
+F test/uri.test 8f27eaa41804099fca15101e30fd1b29aebbebf32d4e3c24614fa6319216936d
 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7
 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9
 F test/utf16align.test 9fde0bb5d3a821594aa68c6829ab9c5453a084384137ebb9f6153e2d678039da
@@ -1821,8 +1830,20 @@ F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/vtabdistinct.test 7688f0889358f849fd60bbfde1ded38b014b18066076d4bfbb75395804dfe072
 F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12
 F test/vtabrhs1.test 9b5ecbc74a689500c33a4b2b36761f9bcc22fcc4e3f9d21066ee0c9c74cf5f6c
-F test/wal.test b7cc6984709f54afbf8441747ced1f646af120bf0c1b1d847bfa39306fbea089
+F test/wal.test 07d61ee522bc203235a7979645d6c5d84f86882afad5b678532ee37b5b85b133
 F test/wal2.test 31f6e2c404b9f2cdf9ca19b105a1742fdc19653c2c936da39e3658c617524046
+F test/wal2big.test 829141cbecdda4329db8fa38705424c4a73db72a06b9540b06811a825d330409
+F test/wal2fault.test 2e8e60cacd5bcd451618aeffd05f676894d17202d3e2986e288d36e2c5993249
+F test/wal2lock.test 0ef98d72dc6bcf7711dedd684760488400d9a9a6eec0dc5d3822060437793552
+F test/wal2openclose.test 2b26be723ea7f4263c8d5d70b37efd1c359561a0526e39466c45fe8e6478daee
+F test/wal2recover.test ba8f4bc9397c838734619f9e759bd98b00e355347b3cf80a2e677610d231d5d8
+F test/wal2recover2.test 0c46afc759e4392a3c12fba17432b880c93a13bf4246d1be5101b00bae4c5f01
+F test/wal2recover3.test 4a91689e165a38bc401736e6518188c2b0ff4fa1566d1810b8867536db128177
+F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c
+F test/wal2rollback.test 23adc4a099b23f6aaea8b04fdca1c35861d887dd80f8be7da2d5273eb777e428
+F test/wal2savepoint.test 2c82bd6a6ee5066c156040d2e9c2415646fcf96116ae7ad127eaf0c0b4a85f22
+F test/wal2simple.test 320a08927f307d0ead26fa3fcef4e3f64279f49be9504f918cb33294f75aeec8
+F test/wal2snapshot.test 7a5f4629a3c43a43c3440b8b2ea9f07de91a46b0b9eea5f08f62b5bf5b6468df
 F test/wal3.test 5de023bb862fd1eb9d2ad26fa8d9c43abb5370582e5b08b2ae0d6f93661bc310
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1848,10 +1869,11 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8d78a1ec91299163451417b451a2bac3481f8eb9f455b1ca507a6625c927ca6e
 F test/walprotocol.test 1b3f922125e341703f6e946d77fdc564d38fb3e07a9385cfdc6c99cac1ecf878
-F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db868eebc131
+F test/walprotocol2.test 7e4bedd5ee83607e2928ac438bf7332a396b980d3e02aa0746509ce11ad1f13c
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
 F test/walro2.test 33955a6fd874dd9724005e17f77fef89d334b3171454a1256fe4941a96766cdc
 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
+F test/walseh1.test 5971f753bd3504ac485ef2dd228894f6bbd20f6711f8e6ee65d62c7dc1a134c8
 F test/walsetlk.test 34c901443b31ab720afc463f5b236c86ca5c4134402573dce91aa0761de8db5a
 F test/walshared.test 42e3808582504878af237ea02c42ca793e8a0efaa19df7df26ac573370dbc7a3
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
@@ -1865,7 +1887,7 @@ F test/where3.test 5b4ffc0ac2ea0fe92f02b1244b7531522fe4d7bccf6fa8741d54e82c10e67
 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8
 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b
-F test/where7.test 1c1bf436bf31b913d4764a2b62ac6e98b9681e5c7ae2b562605592a56b7e946b
+F test/where7.test 15041c7a5838f3bac98f3fb933709674a0b59367664e88fafaf105ff7416eb07
 F test/where8.test 461ca40265ed996a6305da99bb024b0e41602bb586acf544c08f95922358e49f
 F test/where9.test 2db942671a002621eff4f713e347bb25243295f79d8990297cd160bebcfde3f7
 F test/whereA.test 9d1077b117f1b68d5f739d94f36956c36cf995eb87bb19b77b2e81af020edd20
@@ -1939,7 +1961,7 @@ F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb
 F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e04534bbf
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
 F tool/build-all-msvc.bat c817b716e0edeecaf265a6775b63e5f45c34a6544f1d4114a222701ed5ac79ab x
-F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
+F tool/build-shell.sh f193b5e3eb4afcb4abbf96bf1475be6cfb74763ee2e50c82bc7ca105e8a136c5
 F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x
 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2
 F tool/custom.txt 6cdf298f43e1db4bb91406d14777669b8fb1df790837823fa6754c4308decc27
@@ -1966,7 +1988,7 @@ F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439
 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176
 F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8
 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x
-F tool/mkctimec.tcl 38e3db33210a200aae791635125052a643a27aa0619a0debf19aa9c55e1b2dde x
+F tool/mkctimec.tcl 64c69c774528f23d623b3b295f4714b49d2b9b44ec0f282b617a3816f1b94bf6 x
 F tool/mkkeywordhash.c b9faa0ae7e14e4dbbcd951cddd786bf46b8a65bb07b129ba8c0cfade723aaffd
 F tool/mkmsvcmin.tcl 8897d515ef7f94772322db95a3b6fce6c614d84fe0bdd06ba5a1c786351d5a1d
 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef
@@ -1977,7 +1999,7 @@ F tool/mkshellc.tcl b7adf08b82de60811d2cb6af05ff59fc17e5cd6f3e98743c14eaaa3f8971
 F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl 4f7cfef5152b0c91920355cbfc1d608a4ad242cb819f1aea07f6d0274f584a7f
-F tool/mksqlite3c.tcl 44bc3bdb675a9e3a507f4d82575332bccd7e6f830a47c9aa29460ea2240593a9
+F tool/mksqlite3c.tcl 49e39b1e616abc92fd8c24445f2b0a38881825f764541e0026f72371f0d84b65
 F tool/mksqlite3h.tcl d391cff7cad0a372ee1406faee9ccc7dad9cb80a0c95cae0f73d10dd26e06762
 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b
 F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5
@@ -2020,7 +2042,7 @@ F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003
 F tool/vdbe-compress.tcl 1dcb7632e57cf57105248029e6e162fddaf6c0fccb3bb9e6215603752c5a2d4a
 F tool/vdbe_profile.tcl 3ac5a4a9449f4baf77059358ea050db3e34395ccf59c5464d29b91746d5b961e
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
-F tool/warnings.sh ab651bb82586c43ff8b560beceac959735bf917b44c5e0f67ba3426e474f29f8
+F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
 F vsixtest/App.xaml b76d3b48860e7454775c47ea38ffea9c4abe3e85
 F vsixtest/App.xaml.cpp 41158ee43269820136fa3bba00c0bd91b26cc38b650ee392aec2a8d823e54318
@@ -2043,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 06f829e9e57c5aa495f519ad7bb379be611e8e21ef60d36e63d055e45f7a6117
-R f650c894c3b7e9fe01bf709dbdf65d35
-U drh
-Z ff67cbf0340dd6d2b6c094e434a4b95a
+P fd58c616244dc70a2d6c17b62830398cea8177366828260882df5ab1f361a347
+R 8c982794323889a4cb5909db4c119431
+U dan
+Z fd542a929c1ac4ec2e467c191acdf6ca
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-414a4d660c3b505640227066b468db1268fcb1bd3fad931644a769617c6c00cd
+87242777ec869d698c5c78283b98677105e8b5a8b0d8b71a696b8460d4303783

src/btree.c

@@ -3261,10 +3261,10 @@ static int lockBtree(BtShared *pBt){
       goto page1_init_failed;
     }
 #else
-    if( page1[18]>2 ){
+    if( page1[18]>3 ){
       pBt->btsFlags |= BTS_READ_ONLY;
     }
-    if( page1[19]>2 ){
+    if( page1[19]>3 ){
       goto page1_init_failed;
     }
 
@@ -3276,9 +3276,9 @@ static int lockBtree(BtShared *pBt){
     ** may not be the latest version - there may be a newer one in the log
     ** file.
     */
-    if( page1[19]==2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){
+    if( page1[19]>=2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){
       int isOpen = 0;
-      rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen);
+      rc = sqlite3PagerOpenWal(pBt->pPager, (page1[19]==3), &isOpen);
       if( rc!=SQLITE_OK ){
         goto page1_init_failed;
       }else{
@@ -4826,7 +4826,6 @@ void sqlite3BtreeCursorUnpin(BtCursor *pCur){
   pCur->curFlags &= ~BTCF_Pinned;
 }
 
-#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
 /*
 ** Return the offset into the database file for the start of the
 ** payload to which the cursor is pointing.
@@ -4838,7 +4837,6 @@ i64 sqlite3BtreeOffset(BtCursor *pCur){
   return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) +
          (i64)(pCur->info.pPayload - pCur->pPage->aData);
 }
-#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */
 
 /*
 ** Return the number of bytes of payload for the entry that pCur is
@@ -11304,7 +11302,7 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){
   BtShared *pBt = pBtree->pBt;
   int rc;                         /* Return code */
 
-  assert( iVersion==1 || iVersion==2 );
+  assert( iVersion==1 || iVersion==2 || iVersion==3 );
 
   /* If setting the version fields to 1, do not automatically open the
   ** WAL connection, even if the version fields are currently set to 2.

src/btree.h

@@ -321,9 +321,7 @@ int sqlite3BtreePrevious(BtCursor*, int flags);
 i64 sqlite3BtreeIntegerKey(BtCursor*);
 void sqlite3BtreeCursorPin(BtCursor*);
 void sqlite3BtreeCursorUnpin(BtCursor*);
-#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
 i64 sqlite3BtreeOffset(BtCursor*);
-#endif
 int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*);
 const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt);
 u32 sqlite3BtreePayloadSize(BtCursor*);

src/btreeInt.h

@@ -180,7 +180,7 @@
 **    0x81 0x00                 becomes  0x00000080
 **    0x82 0x00                 becomes  0x00000100
 **    0x80 0x7f                 becomes  0x0000007f
-**    0x8a 0x91 0xd1 0xac 0x78  becomes  0x12345678
+**    0x81 0x91 0xd1 0xac 0x78  becomes  0x12345678
 **    0x81 0x81 0x81 0x81 0x01  becomes  0x10204081
 **
 ** Variable length integers are used for rowids and to hold the number of

src/ctime.c

@@ -770,6 +770,7 @@ static const char * const sqlite3azCompileOpt[] = {
 #ifdef SQLITE_VDBE_COVERAGE
   "VDBE_COVERAGE",
 #endif
+"WAL2",
 #ifdef SQLITE_WIN32_MALLOC
   "WIN32_MALLOC",
 #endif

src/delete.c

@@ -525,7 +525,8 @@ void sqlite3DeleteFrom(
     if( pWInfo==0 ) goto delete_from_cleanup;
     eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
     assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI );
-    assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF );
+    assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF
+            || OptimizationDisabled(db, SQLITE_OnePass) );
     if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse);
     if( sqlite3WhereUsesDeferredSeek(pWInfo) ){
       sqlite3VdbeAddOp1(v, OP_FinishSeek, iTabCur);

src/expr.c

@@ -3972,10 +3972,13 @@ int sqlite3ExprCodeGetColumn(
   u8 p5            /* P5 value for OP_Column + FLAGS */
 ){
   assert( pParse->pVdbe!=0 );
+  assert( (p5 & (OPFLAG_NOCHNG|OPFLAG_TYPEOFARG|OPFLAG_LENGTHARG))==p5 );
+  assert( IsVirtual(pTab) || (p5 & OPFLAG_NOCHNG)==0 );
   sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg);
   if( p5 ){
     VdbeOp *pOp = sqlite3VdbeGetLastOp(pParse->pVdbe);
     if( pOp->opcode==OP_Column ) pOp->p5 = p5;
+    if( pOp->opcode==OP_VColumn ) pOp->p5 = (p5 & OPFLAG_NOCHNG);
   }
   return iReg;
 }

src/func.c

@@ -1227,6 +1227,7 @@ static void charFunc(
       *zOut++ = 0x80 + (u8)(c & 0x3F);
     }                                                    \
   }
+  *zOut = 0;
   sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8);
 }
 

src/main.c

@@ -1665,9 +1665,9 @@ static int sqliteDefaultBusyCallback(
   void *ptr,               /* Database connection */
   int count                /* Number of times table has been busy */
 ){
-#if SQLITE_OS_WIN || HAVE_USLEEP
+#if SQLITE_OS_WIN || !defined(HAVE_NANOSLEEP) || HAVE_NANOSLEEP
   /* This case is for systems that have support for sleeping for fractions of
-  ** a second.  Examples:  All windows systems, unix systems with usleep() */
+  ** a second.  Examples:  All windows systems, unix systems with nanosleep() */
   static const u8 delays[] =
      { 1, 2, 5, 10, 15, 20, 25, 25,  25,  50,  50, 100 };
   static const u8 totals[] =
@@ -4160,10 +4160,12 @@ int sqlite3_test_control(int op, ...){
         sqlite3ShowSrcList(0);
         sqlite3ShowWith(0);
         sqlite3ShowUpsert(0);
+#ifndef SQLITE_OMIT_TRIGGER
         sqlite3ShowTriggerStep(0);
         sqlite3ShowTriggerStepList(0);
         sqlite3ShowTrigger(0);
         sqlite3ShowTriggerList(0);
+#endif
 #ifndef SQLITE_OMIT_WINDOWFUNC
         sqlite3ShowWindow(0);
         sqlite3ShowWinFunc(0);
@@ -4470,6 +4472,7 @@ int sqlite3_test_control(int op, ...){
       break;
     }
 
+#if !defined(SQLITE_OMIT_WSD)
     /* sqlite3_test_control(SQLITE_TESTCTRL_USELONGDOUBLE, int X);
     **
     **   X<0     Make no changes to the bUseLongDouble.  Just report value.
@@ -4484,6 +4487,7 @@ int sqlite3_test_control(int op, ...){
       rc = sqlite3Config.bUseLongDouble!=0;
       break;
     }
+#endif
 
 
 #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD)

src/os_unix.c

@@ -6715,12 +6715,17 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){
 ** than the argument.
 */
 static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){
-#if OS_VXWORKS || _POSIX_C_SOURCE >= 199309L
+#if !defined(HAVE_NANOSLEEP) || HAVE_NANOSLEEP+0
   struct timespec sp;
-
   sp.tv_sec = microseconds / 1000000;
   sp.tv_nsec = (microseconds % 1000000) * 1000;
+
+  /* Almost all modern unix systems support nanosleep().  But if you are
+  ** compiling for one of the rare exceptions, you can use
+  ** -DHAVE_NANOSLEEP=0 (perhaps in conjuction with -DHAVE_USLEEP if
+  ** usleep() is available) in order to bypass the use of nanosleep() */
   nanosleep(&sp, NULL);
+
   UNUSED_PARAMETER(NotUsed);
   return microseconds;
 #elif defined(HAVE_USLEEP) && HAVE_USLEEP

src/pager.c

@@ -789,20 +789,6 @@ static const unsigned char aJournalMagic[] = {
 # define USEFETCH(x) 0
 #endif
 
-/*
-** The argument to this macro is a file descriptor (type sqlite3_file*).
-** Return 0 if it is not open, or non-zero (but not 1) if it is.
-**
-** This is so that expressions can be written as:
-**
-**   if( isOpen(pPager->jfd) ){ ...
-**
-** instead of
-**
-**   if( pPager->jfd->pMethods ){ ...
-*/
-#define isOpen(pFd) ((pFd)->pMethods!=0)
-
 #ifdef SQLITE_DIRECT_OVERFLOW_READ
 /*
 ** Return true if page pgno can be read directly from the database file
@@ -934,6 +920,7 @@ static int assert_pager_state(Pager *p){
         assert( isOpen(p->jfd)
              || p->journalMode==PAGER_JOURNALMODE_OFF
              || p->journalMode==PAGER_JOURNALMODE_WAL
+             || p->journalMode==PAGER_JOURNALMODE_WAL2
         );
       }
       assert( pPager->dbOrigSize==pPager->dbFileSize );
@@ -948,6 +935,7 @@ static int assert_pager_state(Pager *p){
       assert( isOpen(p->jfd)
            || p->journalMode==PAGER_JOURNALMODE_OFF
            || p->journalMode==PAGER_JOURNALMODE_WAL
+           || p->journalMode==PAGER_JOURNALMODE_WAL2
            || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
       );
       assert( pPager->dbOrigSize<=pPager->dbHintSize );
@@ -960,6 +948,7 @@ static int assert_pager_state(Pager *p){
       assert( isOpen(p->jfd)
            || p->journalMode==PAGER_JOURNALMODE_OFF
            || p->journalMode==PAGER_JOURNALMODE_WAL
+           || p->journalMode==PAGER_JOURNALMODE_WAL2
            || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
       );
       break;
@@ -2058,7 +2047,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){
       }
       pPager->journalOff = 0;
     }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST
-      || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL)
+      || (pPager->exclusiveMode && pPager->journalMode<PAGER_JOURNALMODE_WAL)
     ){
       rc = zeroJournalHdr(pPager, hasSuper||pPager->tempFile);
       pPager->journalOff = 0;
@@ -2073,6 +2062,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){
       assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE
            || pPager->journalMode==PAGER_JOURNALMODE_MEMORY
            || pPager->journalMode==PAGER_JOURNALMODE_WAL
+           || pPager->journalMode==PAGER_JOURNALMODE_WAL2
       );
       sqlite3OsClose(pPager->jfd);
       if( bDelete ){
@@ -3210,6 +3200,10 @@ static int pagerBeginReadTransaction(Pager *pPager){
   if( rc!=SQLITE_OK || changed ){
     pager_reset(pPager);
     if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0);
+    assert( pPager->journalMode==PAGER_JOURNALMODE_WAL
+         || pPager->journalMode==PAGER_JOURNALMODE_WAL2
+    );
+    pPager->journalMode = sqlite3WalJournalMode(pPager->pWal);
   }
 
   return rc;
@@ -3305,9 +3299,9 @@ static int pagerOpenWalIfPresent(Pager *pPager){
           rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0);
         }else{
           testcase( sqlite3PcachePagecount(pPager->pPCache)==0 );
-          rc = sqlite3PagerOpenWal(pPager, 0);
+          rc = sqlite3PagerOpenWal(pPager, 0, 0);
         }
-      }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){
+      }else if( pPager->journalMode>=PAGER_JOURNALMODE_WAL ){
         pPager->journalMode = PAGER_JOURNALMODE_DELETE;
       }
     }
@@ -4811,6 +4805,7 @@ int sqlite3PagerOpen(
     nPathname + 8 + 1 +                  /* Journal filename */
 #ifndef SQLITE_OMIT_WAL
     nPathname + 4 + 1 +                  /* WAL filename */
+    nPathname + 5 + 1 +                  /* Second WAL filename */
 #endif
     3                                    /* Terminator */
   );
@@ -4863,6 +4858,8 @@ int sqlite3PagerOpen(
     sqlite3FileSuffix3(zFilename, pPager->zWal);
     pPtr = (u8*)(pPager->zWal + sqlite3Strlen30(pPager->zWal)+1);
 #endif
+    memcpy(pPtr, zPathname, nPathname);   pPtr += nPathname;
+    memcpy(pPtr, "-wal2", 5);             pPtr += 5 + 1;
   }else{
     pPager->zWal = 0;
   }
@@ -7277,7 +7274,8 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){
             || eMode==PAGER_JOURNALMODE_OFF       /* 2 */
             || eMode==PAGER_JOURNALMODE_TRUNCATE  /* 3 */
             || eMode==PAGER_JOURNALMODE_MEMORY    /* 4 */
-            || eMode==PAGER_JOURNALMODE_WAL       /* 5 */ );
+            || eMode==PAGER_JOURNALMODE_WAL       /* 5 */
+            || eMode==PAGER_JOURNALMODE_WAL2      /* 6 */ );
 
   /* This routine is only called from the OP_JournalMode opcode, and
   ** the logic there will never allow a temporary file to be changed
@@ -7311,9 +7309,12 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){
     assert( (PAGER_JOURNALMODE_MEMORY & 5)==4 );
     assert( (PAGER_JOURNALMODE_OFF & 5)==0 );
     assert( (PAGER_JOURNALMODE_WAL & 5)==5 );
+    assert( (PAGER_JOURNALMODE_WAL2 & 5)==4 );
 
     assert( isOpen(pPager->fd) || pPager->exclusiveMode );
-    if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){
+    if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 
+     && eMode!=PAGER_JOURNALMODE_WAL2       /* TODO: fix this if possible */
+    ){
       /* In this case we would like to delete the journal file. If it is
       ** not possible, then that is not a problem. Deleting the journal file
       ** here is an optimization only.
@@ -7488,7 +7489,7 @@ static int pagerExclusiveLock(Pager *pPager){
 ** lock on the database file and use heap-memory to store the wal-index
 ** in. Otherwise, use the normal shared-memory.
 */
-static int pagerOpenWal(Pager *pPager){
+static int pagerOpenWal(Pager *pPager, int bWal2){
   int rc = SQLITE_OK;
 
   assert( pPager->pWal==0 && pPager->tempFile==0 );
@@ -7509,7 +7510,7 @@ static int pagerOpenWal(Pager *pPager){
   if( rc==SQLITE_OK ){
     rc = sqlite3WalOpen(pPager->pVfs,
         pPager->fd, pPager->zWal, pPager->exclusiveMode,
-        pPager->journalSizeLimit, &pPager->pWal
+        pPager->journalSizeLimit, bWal2, &pPager->pWal
     );
   }
   pagerFixMaplimit(pPager);
@@ -7535,6 +7536,7 @@ static int pagerOpenWal(Pager *pPager){
 */
 int sqlite3PagerOpenWal(
   Pager *pPager,                  /* Pager object */
+  int bWal2,                      /* Open in wal2 mode if not already open */
   int *pbOpen                     /* OUT: Set to true if call is a no-op */
 ){
   int rc = SQLITE_OK;             /* Return code */
@@ -7551,9 +7553,9 @@ int sqlite3PagerOpenWal(
     /* Close any rollback journal previously open */
     sqlite3OsClose(pPager->jfd);
 
-    rc = pagerOpenWal(pPager);
+    rc = pagerOpenWal(pPager, bWal2);
     if( rc==SQLITE_OK ){
-      pPager->journalMode = PAGER_JOURNALMODE_WAL;
+      pPager->journalMode = bWal2?PAGER_JOURNALMODE_WAL2:PAGER_JOURNALMODE_WAL;
       pPager->eState = PAGER_OPEN;
     }
   }else{
@@ -7575,7 +7577,9 @@ int sqlite3PagerOpenWal(
 int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){
   int rc = SQLITE_OK;
 
-  assert( pPager->journalMode==PAGER_JOURNALMODE_WAL );
+  assert( pPager->journalMode==PAGER_JOURNALMODE_WAL 
+       || pPager->journalMode==PAGER_JOURNALMODE_WAL2
+  );
 
   /* If the log file is not already open, but does exist in the file-system,
   ** it may need to be checkpointed before the connection can switch to
@@ -7590,7 +7594,7 @@ int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){
       );
     }
     if( rc==SQLITE_OK && logexists ){
-      rc = pagerOpenWal(pPager);
+      rc = pagerOpenWal(pPager, 0);
     }
   }
    
@@ -7729,4 +7733,10 @@ int sqlite3PagerWalFramesize(Pager *pPager){
 }
 #endif
 
+#ifdef SQLITE_USE_SEH
+int sqlite3PagerWalSystemErrno(Pager *pPager){
+  return sqlite3WalSystemErrno(pPager->pWal);
+}
+#endif
+
 #endif /* SQLITE_OMIT_DISKIO */

src/pager.h

@@ -82,6 +82,23 @@ typedef struct PgHdr DbPage;
 #define PAGER_JOURNALMODE_TRUNCATE    3   /* Commit by truncating journal */
 #define PAGER_JOURNALMODE_MEMORY      4   /* In-memory journal file */
 #define PAGER_JOURNALMODE_WAL         5   /* Use write-ahead logging */
+#define PAGER_JOURNALMODE_WAL2        6   /* Use write-ahead logging mode 2 */
+
+#define isWalMode(x) ((x)==PAGER_JOURNALMODE_WAL || (x)==PAGER_JOURNALMODE_WAL2)
+
+/*
+** The argument to this macro is a file descriptor (type sqlite3_file*).
+** Return 0 if it is not open, or non-zero (but not 1) if it is.
+**
+** This is so that expressions can be written as:
+**
+**   if( isOpen(pPager->jfd) ){ ...
+**
+** instead of
+**
+**   if( pPager->jfd->pMethods ){ ...
+*/
+#define isOpen(pFd) ((pFd)->pMethods!=0)
 
 /*
 ** Flags that make up the mask passed to sqlite3PagerGet().
@@ -175,7 +192,7 @@ int sqlite3PagerSharedLock(Pager *pPager);
   int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*);
   int sqlite3PagerWalSupported(Pager *pPager);
   int sqlite3PagerWalCallback(Pager *pPager);
-  int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen);
+  int sqlite3PagerOpenWal(Pager *pPager, int, int *pisOpen);
   int sqlite3PagerCloseWal(Pager *pPager, sqlite3*);
 # ifdef SQLITE_ENABLE_SNAPSHOT
   int sqlite3PagerSnapshotGet(Pager*, sqlite3_snapshot **ppSnapshot);
@@ -240,4 +257,8 @@ void sqlite3PagerRekey(DbPage*, Pgno, u16);
 # define enable_simulated_io_errors()
 #endif
 
+#ifdef SQLITE_USE_SEH
+int sqlite3PagerWalSystemErrno(Pager*);
+#endif
+
 #endif /* SQLITE_PAGER_H */

src/parse.y

@@ -148,8 +148,8 @@ ecmd ::= SEMI.
 ecmd ::= cmdx SEMI.
 %ifndef SQLITE_OMIT_EXPLAIN
 ecmd ::= explain cmdx SEMI.       {NEVER-REDUCE}
-explain ::= EXPLAIN.              { pParse->explain = 1; }
-explain ::= EXPLAIN QUERY PLAN.   { pParse->explain = 2; }
+explain ::= EXPLAIN.              { if( pParse->pReprepare==0 ) pParse->explain = 1; }
+explain ::= EXPLAIN QUERY PLAN.   { if( pParse->pReprepare==0 ) pParse->explain = 2; }
 %endif  SQLITE_OMIT_EXPLAIN
 cmdx ::= cmd.           { sqlite3FinishCoding(pParse); }
 

src/pragma.c

@@ -262,7 +262,7 @@ const char *sqlite3JournalModename(int eMode){
   static char * const azModeName[] = {
     "delete", "persist", "off", "truncate", "memory"
 #ifndef SQLITE_OMIT_WAL
-     , "wal"
+     , "wal", "wal2"
 #endif
   };
   assert( PAGER_JOURNALMODE_DELETE==0 );
@@ -271,6 +271,7 @@ const char *sqlite3JournalModename(int eMode){
   assert( PAGER_JOURNALMODE_TRUNCATE==3 );
   assert( PAGER_JOURNALMODE_MEMORY==4 );
   assert( PAGER_JOURNALMODE_WAL==5 );
+  assert( PAGER_JOURNALMODE_WAL2==6 );
   assert( eMode>=0 && eMode<=ArraySize(azModeName) );
 
   if( eMode==ArraySize(azModeName) ) return 0;

src/prepare.c

@@ -700,7 +700,12 @@ static int sqlite3Prepare(
   sParse.pOuterParse = db->pParse;
   db->pParse = &sParse;
   sParse.db = db;
-  sParse.pReprepare = pReprepare;
+  if( pReprepare ){
+    sParse.pReprepare = pReprepare;
+    sParse.explain = sqlite3_stmt_isexplain((sqlite3_stmt*)pReprepare);
+  }else{
+    assert( sParse.pReprepare==0 );
+  }
   assert( ppStmt && *ppStmt==0 );
   if( db->mallocFailed ){
     sqlite3ErrorMsg(&sParse, "out of memory");

src/printf.c

@@ -1366,3 +1366,72 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){
   sqlite3_str_vappendf(p, zFormat, ap);
   va_end(ap);
 }
+
+
+/*****************************************************************************
+** Reference counted string storage
+*****************************************************************************/
+
+/*
+** Increase the reference count of the string by one.
+**
+** The input parameter is returned.
+*/
+char *sqlite3RCStrRef(char *z){
+  RCStr *p = (RCStr*)z;
+  assert( p!=0 );
+  p--;
+  p->nRCRef++;
+  return z;
+}
+
+/*
+** Decrease the reference count by one.  Free the string when the
+** reference count reaches zero.
+*/
+void sqlite3RCStrUnref(char *z){
+  RCStr *p = (RCStr*)z;
+  assert( p!=0 );
+  p--;
+  assert( p->nRCRef>0 );
+  if( p->nRCRef>=2 ){
+    p->nRCRef--;
+  }else{
+    sqlite3_free(p);
+  }
+}
+
+/*
+** Create a new string that is capable of holding N bytes of text, not counting
+** the zero byte at the end.  The string is uninitialized.
+**
+** The reference count is initially 1.  Call sqlite3RCStrUnref() to free the
+** newly allocated string.
+**
+** This routine returns 0 on an OOM.
+*/
+char *sqlite3RCStrNew(u64 N){
+  RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 );
+  if( p==0 ) return 0;
+  p->nRCRef = 1;
+  return (char*)&p[1];
+}
+
+/*
+** Change the size of the string so that it is able to hold N bytes.
+** The string might be reallocated, so return the new allocation.
+*/
+char *sqlite3RCStrResize(char *z, u64 N){
+  RCStr *p = (RCStr*)z;
+  RCStr *pNew;
+  assert( p!=0 );
+  p--;
+  assert( p->nRCRef==1 );
+  pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1);
+  if( pNew==0 ){
+    sqlite3_free(p);
+    return 0;
+  }else{
+    return (char*)&pNew[1];
+  }
+}

src/select.c

@@ -1298,6 +1298,16 @@ static void selectInnerLoop(
       testcase( eDest==SRT_Fifo );
       testcase( eDest==SRT_DistFifo );
       sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg);
+#if !defined(SQLITE_ENABLE_NULL_TRIM) && defined(SQLITE_DEBUG)
+      /* A destination of SRT_Table and a non-zero iSDParm2 parameter means
+      ** that this is an "UPDATE ... FROM" on a virtual table or view. In this
+      ** case set the p5 parameter of the OP_MakeRecord to OPFLAG_NOCHNG_MAGIC.
+      ** This does not affect operation in any way - it just allows MakeRecord
+      ** to process OPFLAG_NOCHANGE values without an assert() failing. */
+      if( eDest==SRT_Table && pDest->iSDParm2 ){
+        sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC);
+      }
+#endif
 #ifndef SQLITE_OMIT_CTE
       if( eDest==SRT_DistFifo ){
         /* If the destination is DistFifo, then cursor (iParm+1) is open
@@ -2101,13 +2111,6 @@ void sqlite3GenerateColumnNames(
   int fullName;    /* TABLE.COLUMN if no AS clause and is a direct table ref */
   int srcName;     /* COLUMN or TABLE.COLUMN if no AS clause and is direct */
 
-#ifndef SQLITE_OMIT_EXPLAIN
-  /* If this is an EXPLAIN, skip this step */
-  if( pParse->explain ){
-    return;
-  }
-#endif
-
   if( pParse->colNamesSet ) return;
   /* Column names are determined by the left-most term of a compound select */
   while( pSelect->pPrior ) pSelect = pSelect->pPrior;
@@ -4299,7 +4302,8 @@ static int compoundHasDifferentAffinities(Select *p){
 **        (27b) the subquery is a compound query and the RIGHT JOIN occurs
 **              in any arm of the compound query.  (See also (17g).)
 **
-**  (28)  The subquery is not a MATERIALIZED CTE.
+**  (28)  The subquery is not a MATERIALIZED CTE.  (This is handled
+**        in the caller before ever reaching this routine.)
 **
 **
 ** In this routine, the "p" parameter is a pointer to the outer query.
@@ -4409,9 +4413,9 @@ static int flattenSubquery(
   if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){
     return 0;   /* Restriction (27a) */
   }
-  if( pSubitem->fg.isCte && pSubitem->u2.pCteUse->eM10d==M10d_Yes ){
-    return 0;       /* (28) */
-  }
+
+  /* Condition (28) is blocked by the caller */
+  assert( !pSubitem->fg.isCte || pSubitem->u2.pCteUse->eM10d!=M10d_Yes );
 
   /* Restriction (17): If the sub-query is a compound SELECT, then it must
   ** use only the UNION ALL operator. And none of the simple select queries
@@ -7292,6 +7296,14 @@ int sqlite3Select(
       goto select_end;
     }
 
+    /* Do not attempt the usual optimizations (flattening and ORDER BY
+    ** elimination) on a MATERIALIZED common table expression because
+    ** a MATERIALIZED common table expression is an optimization fence.
+    */
+    if( pItem->fg.isCte && pItem->u2.pCteUse->eM10d==M10d_Yes ){
+      continue;
+    }
+
     /* Do not try to flatten an aggregate subquery.
     **
     ** Flattening an aggregate subquery is only possible if the outer query
@@ -7321,6 +7333,8 @@ int sqlite3Select(
     **            (a)  The outer query has a different ORDER BY clause
     **            (b)  The subquery is part of a join
     **          See forum post 062d576715d277c8
+    **
+    ** Also retain the ORDER BY if the OmitOrderBy optimization is disabled.
     */
     if( pSub->pOrderBy!=0
      && (p->pOrderBy!=0 || pTabList->nSrc>1)      /* Condition (5) */

src/shell.c.in

@@ -1607,6 +1607,7 @@ static ShellState shellState;
 #define MODE_Box     16  /* Unicode box-drawing characters */
 #define MODE_Count   17  /* Output only a count of the rows of output */
 #define MODE_Off     18  /* No query output shown */
+#define MODE_ScanExp 19  /* Like MODE_Explain, but for ".scanstats vm" */
 
 static const char *modeDescr[] = {
   "line",
@@ -2520,38 +2521,58 @@ static int shell_callback(
       }
       break;
     }
+    case MODE_ScanExp:
     case MODE_Explain: {
-      static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13};
-      if( nArg>ArraySize(aExplainWidth) ){
-        nArg = ArraySize(aExplainWidth);
+      static const int aExplainWidth[] = {4,       13, 4, 4, 4, 13, 2, 13};
+      static const int aExplainMap[] =   {0,       1,  2, 3, 4, 5,  6, 7 };
+      static const int aScanExpWidth[] = {4, 6, 6, 13, 4, 4, 4, 13, 2, 13};
+      static const int aScanExpMap[] =   {0, 9, 8, 1,  2, 3, 4, 5,  6, 7 };
+
+      const int *aWidth = aExplainWidth;
+      const int *aMap = aExplainMap;
+      int nWidth = ArraySize(aExplainWidth);
+      int iIndent = 1;
+
+      if( p->cMode==MODE_ScanExp ){
+        aWidth = aScanExpWidth;
+        aMap = aScanExpMap;
+        nWidth = ArraySize(aScanExpWidth);
+        iIndent = 3;
       }
+      if( nArg>nWidth ) nArg = nWidth;
+
+      /* If this is the first row seen, print out the headers */
       if( p->cnt++==0 ){
         for(i=0; i<nArg; i++){
-          int w = aExplainWidth[i];
-          utf8_width_print(p->out, w, azCol[i]);
+          utf8_width_print(p->out, aWidth[i], azCol[ aMap[i] ]);
           fputs(i==nArg-1 ? "\n" : "  ", p->out);
         }
         for(i=0; i<nArg; i++){
-          int w = aExplainWidth[i];
-          print_dashes(p->out, w);
+          print_dashes(p->out, aWidth[i]);
           fputs(i==nArg-1 ? "\n" : "  ", p->out);
         }
       }
+
+      /* If there is no data, exit early. */
       if( azArg==0 ) break;
+
       for(i=0; i<nArg; i++){
-        int w = aExplainWidth[i];
+        const char *zSep = "  ";
+        int w = aWidth[i];
+        const char *zVal = azArg[ aMap[i] ];
         if( i==nArg-1 ) w = 0;
-        if( azArg[i] && strlenChar(azArg[i])>w ){
-          w = strlenChar(azArg[i]);
+        if( zVal && strlenChar(zVal)>w ){
+          w = strlenChar(zVal);
+          zSep = " ";
         }
-        if( i==1 && p->aiIndent && p->pStmt ){
+        if( i==iIndent && p->aiIndent && p->pStmt ){
           if( p->iIndent<p->nIndent ){
             utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], "");
           }
           p->iIndent++;
         }
-        utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue);
-        fputs(i==nArg-1 ? "\n" : "  ", p->out);
+        utf8_width_print(p->out, w, zVal ? zVal : p->nullValue);
+        fputs(i==nArg-1 ? "\n" : zSep, p->out);
       }
       break;
     }
@@ -3332,17 +3353,11 @@ static int scanStatsHeight(sqlite3_stmt *p, int iEntry){
 }
 #endif
 
-/*
-** Display scan stats.
-*/
-static void display_scanstats(
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+static void display_explain_scanstats(
   sqlite3 *db,                    /* Database to query */
   ShellState *pArg                /* Pointer to ShellState */
 ){
-#ifndef SQLITE_ENABLE_STMT_SCANSTATUS
-  UNUSED_PARAMETER(db);
-  UNUSED_PARAMETER(pArg);
-#else
   static const int f = SQLITE_SCANSTAT_COMPLEX;
   sqlite3_stmt *p = pArg->pStmt;
   int ii = 0;
@@ -3414,8 +3429,9 @@ static void display_scanstats(
   }
 
   eqp_render(pArg, nTotal);
-#endif
 }
+#endif
+
 
 /*
 ** Parameter azArray points to a zero-terminated array of strings. zStr
@@ -3453,8 +3469,6 @@ static int str_in_array(const char *zStr, const char **azArray){
 **       and "Goto" by 2 spaces.
 */
 static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){
-  const char *zSql;               /* The text of the SQL statement */
-  const char *z;                  /* Used to check if this is an EXPLAIN */
   int *abYield = 0;               /* True if op is an OP_Yield */
   int nAlloc = 0;                 /* Allocated size of p->aiIndent[], abYield */
   int iOp;                        /* Index of operation in p->aiIndent[] */
@@ -3465,65 +3479,45 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){
                             "Rewind", 0 };
   const char *azGoto[] = { "Goto", 0 };
 
-  /* Try to figure out if this is really an EXPLAIN statement. If this
-  ** cannot be verified, return early.  */
-  if( sqlite3_column_count(pSql)!=8 ){
-    p->cMode = p->mode;
-    return;
-  }
-  zSql = sqlite3_sql(pSql);
-  if( zSql==0 ) return;
-  for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++);
-  if( sqlite3_strnicmp(z, "explain", 7) ){
-    p->cMode = p->mode;
-    return;
-  }
+  /* The caller guarantees that the leftmost 4 columns of the statement
+  ** passed to this function are equivalent to the leftmost 4 columns
+  ** of EXPLAIN statement output. In practice the statement may be
+  ** an EXPLAIN, or it may be a query on the bytecode() virtual table.  */
+  assert( sqlite3_column_count(pSql)>=4 );
+  assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) );
+  assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) );
+  assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) );
+  assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) );
 
   for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){
     int i;
     int iAddr = sqlite3_column_int(pSql, 0);
     const char *zOp = (const char*)sqlite3_column_text(pSql, 1);
-
-    /* Set p2 to the P2 field of the current opcode. Then, assuming that
-    ** p2 is an instruction address, set variable p2op to the index of that
-    ** instruction in the aiIndent[] array. p2 and p2op may be different if
-    ** the current instruction is part of a sub-program generated by an
-    ** SQL trigger or foreign key.  */
+    int p1 = sqlite3_column_int(pSql, 2);
     int p2 = sqlite3_column_int(pSql, 3);
+
+    /* Assuming that p2 is an instruction address, set variable p2op to the
+    ** index of that instruction in the aiIndent[] array. p2 and p2op may be
+    ** different if the current instruction is part of a sub-program generated
+    ** by an SQL trigger or foreign key.  */
     int p2op = (p2 + (iOp-iAddr));
 
     /* Grow the p->aiIndent array as required */
     if( iOp>=nAlloc ){
-      if( iOp==0 ){
-        /* Do further verification that this is explain output.  Abort if
-        ** it is not */
-        static const char *explainCols[] = {
-           "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment" };
-        int jj;
-        for(jj=0; jj<ArraySize(explainCols); jj++){
-          if( cli_strcmp(sqlite3_column_name(pSql,jj),explainCols[jj])!=0 ){
-            p->cMode = p->mode;
-            sqlite3_reset(pSql);
-            return;
-          }
-        }
-      }
       nAlloc += 100;
       p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int));
       shell_check_oom(p->aiIndent);
       abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int));
       shell_check_oom(abYield);
     }
+
     abYield[iOp] = str_in_array(zOp, azYield);
     p->aiIndent[iOp] = 0;
     p->nIndent = iOp+1;
-
     if( str_in_array(zOp, azNext) && p2op>0 ){
       for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2;
     }
-    if( str_in_array(zOp, azGoto) && p2op<p->nIndent
-     && (abYield[p2op] || sqlite3_column_int(pSql, 2))
-    ){
+    if( str_in_array(zOp, azGoto) && p2op<iOp && (abYield[p2op] || p1) ){
       for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2;
     }
   }
@@ -3543,6 +3537,48 @@ static void explain_data_delete(ShellState *p){
   p->iIndent = 0;
 }
 
+static void exec_prepared_stmt(ShellState*, sqlite3_stmt*);
+
+/*
+** Display scan stats.
+*/
+static void display_scanstats(
+  sqlite3 *db,                    /* Database to query */
+  ShellState *pArg                /* Pointer to ShellState */
+){
+#ifndef SQLITE_ENABLE_STMT_SCANSTATUS
+  UNUSED_PARAMETER(db);
+  UNUSED_PARAMETER(pArg);
+#else
+  if( pArg->scanstatsOn==3 ){
+    const char *zSql = 
+      "  SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec,"
+      "   round(ncycle*100.0 / (sum(ncycle) OVER ()), 2)||'%' AS cycles"
+      "   FROM bytecode(?)";
+
+    int rc = SQLITE_OK;
+    sqlite3_stmt *pStmt = 0;
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3_stmt *pSave = pArg->pStmt;
+      pArg->pStmt = pStmt;
+      sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0);
+
+      pArg->cnt = 0;
+      pArg->cMode = MODE_ScanExp;
+      explain_data_prepare(pArg, pStmt);
+      exec_prepared_stmt(pArg, pStmt);
+      explain_data_delete(pArg);
+
+      sqlite3_finalize(pStmt);
+      pArg->pStmt = pSave;
+    }
+  }else{
+    display_explain_scanstats(db, pArg);
+  }
+#endif
+}
+
 /*
 ** Disable and restore .wheretrace and .treetrace/.selecttrace settings.
 */
@@ -4332,16 +4368,15 @@ static int shell_exec(
       /* Show the EXPLAIN QUERY PLAN if .eqp is on */
       if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){
         sqlite3_stmt *pExplain;
-        char *zEQP;
         int triggerEQP = 0;
         disable_debug_trace_modes();
         sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP);
         if( pArg->autoEQP>=AUTOEQP_trigger ){
           sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0);
         }
-        zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql);
-        shell_check_oom(zEQP);
-        rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
+        pExplain = pStmt;
+        sqlite3_reset(pExplain);
+        rc = sqlite3_stmt_explain(pExplain, 2);
         if( rc==SQLITE_OK ){
           while( sqlite3_step(pExplain)==SQLITE_ROW ){
             const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3);
@@ -4353,36 +4388,31 @@ static int shell_exec(
           }
           eqp_render(pArg, 0);
         }
-        sqlite3_finalize(pExplain);
-        sqlite3_free(zEQP);
         if( pArg->autoEQP>=AUTOEQP_full ){
           /* Also do an EXPLAIN for ".eqp full" mode */
-          zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql);
-          shell_check_oom(zEQP);
-          rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
+          sqlite3_reset(pExplain);
+          rc = sqlite3_stmt_explain(pExplain, 1);
           if( rc==SQLITE_OK ){
             pArg->cMode = MODE_Explain;
+            assert( sqlite3_stmt_isexplain(pExplain)==1 );
             explain_data_prepare(pArg, pExplain);
             exec_prepared_stmt(pArg, pExplain);
             explain_data_delete(pArg);
           }
-          sqlite3_finalize(pExplain);
-          sqlite3_free(zEQP);
         }
         if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){
           sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0);
-          /* Reprepare pStmt before reactivating trace modes */
-          sqlite3_finalize(pStmt);
-          sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-          if( pArg ) pArg->pStmt = pStmt;
         }
+        sqlite3_reset(pStmt);
+        sqlite3_stmt_explain(pStmt, 0);
         restore_debug_trace_modes();
       }
 
       if( pArg ){
+        int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1);
         pArg->cMode = pArg->mode;
         if( pArg->autoExplain ){
-          if( sqlite3_stmt_isexplain(pStmt)==1 ){
+          if( bIsExplain ){
             pArg->cMode = MODE_Explain;
           }
           if( sqlite3_stmt_isexplain(pStmt)==2 ){
@@ -4392,7 +4422,7 @@ static int shell_exec(
 
         /* If the shell is currently in ".explain" mode, gather the extra
         ** data required to add indents to the output.*/
-        if( pArg->cMode==MODE_Explain ){
+        if( pArg->cMode==MODE_Explain && bIsExplain ){
           explain_data_prepare(pArg, pStmt);
         }
       }
@@ -9933,6 +9963,9 @@ static int do_meta_command(char *zLine, ShellState *p){
 
   if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){
     if( nArg==2 ){
+      if( cli_strcmp(azArg[1], "vm")==0 ){
+        p->scanstatsOn = 3;
+      }else
       if( cli_strcmp(azArg[1], "est")==0 ){
         p->scanstatsOn = 2;
       }else{

src/sqlite.h.in

@@ -528,6 +528,7 @@ int sqlite3_exec(
 #define SQLITE_IOERR_ROLLBACK_ATOMIC   (SQLITE_IOERR | (31<<8))
 #define SQLITE_IOERR_DATA              (SQLITE_IOERR | (32<<8))
 #define SQLITE_IOERR_CORRUPTFS         (SQLITE_IOERR | (33<<8))
+#define SQLITE_IOERR_IN_PAGE           (SQLITE_IOERR | (34<<8))
 #define SQLITE_LOCKED_SHAREDCACHE      (SQLITE_LOCKED |  (1<<8))
 #define SQLITE_LOCKED_VTAB             (SQLITE_LOCKED |  (2<<8))
 #define SQLITE_BUSY_RECOVERY           (SQLITE_BUSY   |  (1<<8))
@@ -4421,6 +4422,41 @@ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
 */
 int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt);
 
+/*
+** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement
+** METHOD: sqlite3_stmt
+**
+** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN
+** setting for [prepared statement] S.  If E is zero, then S becomes
+** a normal prepared statement.  If E is 1, then S behaves as if
+** its SQL text began with "[EXPLAIN]".  If E is 2, then S behaves as if
+** its SQL text began with "[EXPLAIN QUERY PLAN]".
+**
+** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared.
+** SQLite tries to avoid a reprepare, but a reprepare might be necessary
+** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode.
+**
+** Because of the potential need to reprepare, a call to
+** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be
+** reprepared because it was created using [sqlite3_prepare()] instead of
+** the newer [sqlite3_prepare_v2()] or [sqlite3_prepare_v3()] interfaces and
+** hence has no saved SQL text with which to reprepare.
+**
+** Changing the explain setting for a prepared statement does not change
+** the original SQL text for the statement.  Hence, if the SQL text originally
+** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0)
+** is called to convert the statement into an ordinary statement, the EXPLAIN
+** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S)
+** output, even though the statement now acts like a normal SQL statement.
+**
+** This routine returns SQLITE_OK if the explain mode is successfully
+** changed, or an error code if the explain mode could not be changed.
+** The explain mode cannot be changed while a statement is active.
+** Hence, it is good practice to call [sqlite3_reset(S)]
+** immediately prior to calling sqlite3_stmt_explain(S,E).
+*/
+int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode);
+
 /*
 ** CAPI3REF: Determine If A Prepared Statement Has Been Reset
 ** METHOD: sqlite3_stmt

src/sqliteInt.h

@@ -894,7 +894,7 @@ typedef INT16_TYPE LogEst;
 /*
 ** P is one byte past the end of a large buffer. Return true if a span of bytes
 ** between S..E crosses the end of that buffer.  In other words, return true
-** if the sub-buffer S..E-1 overflows the buffer show last byte is P-1.
+** if the sub-buffer S..E-1 overflows the buffer whose last byte is P-1.
 **
 ** S is the start of the span.  E is one byte past the end of end of span.
 **
@@ -1257,7 +1257,6 @@ typedef struct Cte Cte;
 typedef struct CteUse CteUse;
 typedef struct Db Db;
 typedef struct DbFixer DbFixer;
-typedef struct DblDbl DblDbl;
 typedef struct Schema Schema;
 typedef struct Expr Expr;
 typedef struct ExprList ExprList;
@@ -1281,6 +1280,7 @@ typedef struct Parse Parse;
 typedef struct ParseCleanup ParseCleanup;
 typedef struct PreUpdate PreUpdate;
 typedef struct PrintfArguments PrintfArguments;
+typedef struct RCStr RCStr;
 typedef struct RenameToken RenameToken;
 typedef struct Returning Returning;
 typedef struct RowSet RowSet;
@@ -1873,6 +1873,7 @@ struct sqlite3 {
 #define SQLITE_IndexedExpr    0x01000000 /* Pull exprs from index when able */
 #define SQLITE_Coroutines     0x02000000 /* Co-routines for subqueries */
 #define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */
+#define SQLITE_OnePass        0x08000000 /* Single-pass DELETE and UPDATE */
 #define SQLITE_AllOpts        0xffffffff /* All optimizations */
 
 /*
@@ -3776,6 +3777,9 @@ struct Parse {
   int regRoot;         /* Register holding root page number for new objects */
   int nMaxArg;         /* Max args passed to user function by sub-program */
   int nSelect;         /* Number of SELECT stmts. Counter for Select.selId */
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+  u32 nProgressSteps;  /* xProgress steps taken during sqlite3_prepare() */
+#endif
 #ifndef SQLITE_OMIT_SHARED_CACHE
   int nTableLock;        /* Number of locks in aTableLock */
   TableLock *aTableLock; /* Required table locks for shared-cache mode */
@@ -3789,12 +3793,9 @@ struct Parse {
     int addrCrTab;         /* Address of OP_CreateBtree on CREATE TABLE */
     Returning *pReturning; /* The RETURNING clause */
   } u1;
-  u32 nQueryLoop;      /* Est number of iterations of a query (10*log2(N)) */
   u32 oldmask;         /* Mask of old.* columns referenced */
   u32 newmask;         /* Mask of new.* columns referenced */
-#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
-  u32 nProgressSteps;  /* xProgress steps taken during sqlite3_prepare() */
-#endif
+  LogEst nQueryLoop;   /* Est number of iterations of a query (10*log2(N)) */
   u8 eTriggerOp;       /* TK_UPDATE, TK_INSERT or TK_DELETE */
   u8 bReturning;       /* Coding a RETURNING trigger */
   u8 eOrconf;          /* Default ON CONFLICT policy for trigger steps */
@@ -4061,6 +4062,25 @@ struct sqlite3_str {
 
 #define isMalloced(X)  (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0)
 
+/*
+** The following object is the header for an "RCStr" or "reference-counted
+** string".  An RCStr is passed around and used like any other char*
+** that has been dynamically allocated.  The important interface
+** differences:
+**
+**   1.  RCStr strings are reference counted.  They are deallocated
+**       when the reference count reaches zero.
+**
+**   2.  Use sqlite3RCStrUnref() to free an RCStr string rather than
+**       sqlite3_free()
+**
+**   3.  Make a (read-only) copy of a read-only RCStr string using
+**       sqlite3RCStrRef().
+*/
+struct RCStr {
+  u64 nRCRef;            /* Number of references */
+  /* Total structure size should be a multiple of 8 bytes for alignment */
+};
 
 /*
 ** A pointer to this structure is used to communicate information
@@ -5180,6 +5200,7 @@ void sqlite3FileSuffix3(const char*, char*);
 u8 sqlite3GetBoolean(const char *z,u8);
 
 const void *sqlite3ValueText(sqlite3_value*, u8);
+int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*));
 int sqlite3ValueBytes(sqlite3_value*, u8);
 void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8,
                         void(*)(void*));
@@ -5287,6 +5308,11 @@ void sqlite3OomClear(sqlite3*);
 int sqlite3ApiExit(sqlite3 *db, int);
 int sqlite3OpenTempDatabase(Parse *);
 
+char *sqlite3RCStrRef(char*);
+void sqlite3RCStrUnref(char*);
+char *sqlite3RCStrNew(u64);
+char *sqlite3RCStrResize(char*,u64);
+
 void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int);
 int sqlite3StrAccumEnlarge(StrAccum*, i64);
 char *sqlite3StrAccumFinish(StrAccum*);

src/test1.c

@@ -2922,6 +2922,34 @@ static int SQLITE_TCLAPI test_stmt_isexplain(
   return TCL_OK;
 }
 
+/*
+** Usage:  sqlite3_stmt_explain  STMT  INT
+**
+** Set the explain to normal (0), EXPLAIN (1) or EXPLAIN QUERY PLAN (2).
+*/
+static int SQLITE_TCLAPI test_stmt_explain(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3_stmt *pStmt;
+  int eMode = 0;
+  int rc;
+
+  if( objc!=3 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"",
+        Tcl_GetStringFromObj(objv[0], 0), " STMT INT", 0);
+    return TCL_ERROR;
+  }
+
+  if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
+  if( Tcl_GetIntFromObj(interp, objv[2], &eMode) ) return TCL_ERROR;
+  rc = sqlite3_stmt_explain(pStmt, eMode);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
+  return TCL_OK;
+}
+
 /*
 ** Usage:  sqlite3_stmt_busy  STMT
 **
@@ -7981,6 +8009,7 @@ static int SQLITE_TCLAPI optimization_control(
     { "push-down",           SQLITE_PushDown       },
     { "balanced-merge",      SQLITE_BalancedMerge  },
     { "propagate-const",     SQLITE_PropagateConst },
+    { "one-pass",            SQLITE_OnePass        },
   };
 
   if( objc!=4 ){
@@ -8991,6 +9020,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_next_stmt",             test_next_stmt     ,0 },
      { "sqlite3_stmt_readonly",         test_stmt_readonly ,0 },
      { "sqlite3_stmt_isexplain",        test_stmt_isexplain,0 },
+     { "sqlite3_stmt_explain",          test_stmt_explain  ,0 },
      { "sqlite3_stmt_busy",             test_stmt_busy     ,0 },
      { "uses_stmt_journal",             uses_stmt_journal ,0 },
 

src/test_hexio.c

@@ -190,7 +190,7 @@ static int SQLITE_TCLAPI hexio_write(
 }
 
 /*
-** USAGE:   hexio_get_int   HEXDATA
+** USAGE:   hexio_get_int [-littleendian] HEXDATA
 **
 ** Interpret the HEXDATA argument as a big-endian integer.  Return
 ** the value of that integer.  HEXDATA can contain between 2 and 8
@@ -207,12 +207,20 @@ static int SQLITE_TCLAPI hexio_get_int(
   const unsigned char *zIn;
   unsigned char *aOut;
   unsigned char aNum[4];
+  int bLittle = 0;
 
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "HEXDATA");
+  if( objc==3 ){
+    int n;
+    char *z = Tcl_GetStringFromObj(objv[1], &n);
+    if( n>=2 && n<=13 && memcmp(z, "-littleendian", n)==0 ){
+      bLittle = 1;
+    }
+  }
+  if( (objc-bLittle)!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "[-littleendian] HEXDATA");
     return TCL_ERROR;
   }
-  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn);
+  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1+bLittle], &nIn);
   aOut = sqlite3_malloc( 1 + nIn/2 );
   if( aOut==0 ){
     return TCL_ERROR;
@@ -225,7 +233,11 @@ static int SQLITE_TCLAPI hexio_get_int(
     memcpy(&aNum[4-nOut], aOut, nOut);
   }
   sqlite3_free(aOut);
-  val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+  if( bLittle ){
+    val = (aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0];
+  }else{
+    val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+  }
   Tcl_SetObjResult(interp, Tcl_NewIntObj(val));
   return TCL_OK;
 }

src/test_tclsh.c

@@ -98,6 +98,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){
   extern int Fts5tcl_Init(Tcl_Interp *);
   extern int SqliteRbu_Init(Tcl_Interp*);
   extern int Sqlitetesttcl_Init(Tcl_Interp*);
+  extern int Bgckpt_Init(Tcl_Interp*);
 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
   extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
 #endif
@@ -167,6 +168,8 @@ const char *sqlite3TestInit(Tcl_Interp *interp){
   Fts5tcl_Init(interp);
   SqliteRbu_Init(interp);
   Sqlitetesttcl_Init(interp);
+  Bgckpt_Init(interp);
+
 
 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
   Sqlitetestfts3_Init(interp);

src/update.c

@@ -741,7 +741,7 @@ void sqlite3Update(
        && !hasFK
        && !chngKey
        && !bReplace
-       && (sNC.ncFlags & NC_Subquery)==0
+       && (pWhere==0 || !ExprHasProperty(pWhere, EP_Subquery))
       ){
         flags |= WHERE_ONEPASS_MULTIROW;
       }
@@ -1259,7 +1259,9 @@ static void updateVirtualTable(
           sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0)
         );
       }else{
-        pList = sqlite3ExprListAppend(pParse, pList, exprRowColumn(pParse, i));
+        Expr *pRow = exprRowColumn(pParse, i);
+        if( pRow ) pRow->op2 = OPFLAG_NOCHNG;
+        pList = sqlite3ExprListAppend(pParse, pList, pRow);
       }
     }
 

src/util.c

@@ -141,6 +141,23 @@ void sqlite3ErrorClear(sqlite3 *db){
 */
 void sqlite3SystemError(sqlite3 *db, int rc){
   if( rc==SQLITE_IOERR_NOMEM ) return;
+#ifdef SQLITE_USE_SEH
+  if( rc==SQLITE_IOERR_IN_PAGE ){
+    int ii;
+    int iErr;
+    sqlite3BtreeEnterAll(db);
+    for(ii=0; ii<db->nDb; ii++){
+      if( db->aDb[ii].pBt ){
+        iErr = sqlite3PagerWalSystemErrno(sqlite3BtreePager(db->aDb[ii].pBt));
+        if( iErr ){
+          db->iSysErrno = iErr;
+        }
+      }
+    }
+    sqlite3BtreeLeaveAll(db);
+    return;
+  }
+#endif
   rc &= 0xff;
   if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){
     db->iSysErrno = sqlite3OsGetLastError(db->pVfs);
@@ -591,7 +608,12 @@ do_atof_calc:
       while( e<=-10  ){ e+=10;  r *= 1.0e-10L;  }
       while( e<=-1   ){ e+=1;   r *= 1.0e-01L;  }
     }
-    *pResult = r;
+    assert( r>=0.0 );
+    if( r>+1.7976931348623157081452742373e+308L ){
+      *pResult = +INFINITY;
+    }else{
+      *pResult = (double)r;
+    }
   }else{
     double rr[2];
     u64 s2;

src/vdbe.c

@@ -695,6 +695,93 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){
   return h;
 }
 
+
+/*
+** For OP_Column, factor out the case where content is loaded from
+** overflow pages, so that the code to implement this case is separate
+** the common case where all content fits on the page.  Factoring out
+** the code reduces register pressure and helps the common case
+** to run faster.
+*/
+static SQLITE_NOINLINE int vdbeColumnFromOverflow(
+  VdbeCursor *pC,       /* The BTree cursor from which we are reading */
+  int iCol,             /* The column to read */
+  int t,                /* The serial-type code for the column value */
+  i64 iOffset,          /* Offset to the start of the content value */
+  u32 cacheStatus,      /* Current Vdbe.cacheCtr value */
+  u32 colCacheCtr,      /* Current value of the column cache counter */
+  Mem *pDest            /* Store the value into this register. */
+){
+  int rc;
+  sqlite3 *db = pDest->db;
+  int encoding = pDest->enc;
+  int len = sqlite3VdbeSerialTypeLen(t);
+  assert( pC->eCurType==CURTYPE_BTREE );
+  if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) return SQLITE_TOOBIG;
+  if( len > 4000 && pC->pKeyInfo==0 ){
+    /* Cache large column values that are on overflow pages using
+    ** an RCStr (reference counted string) so that if they are reloaded,
+    ** that do not have to be copied a second time.  The overhead of
+    ** creating and managing the cache is such that this is only
+    ** profitable for larger TEXT and BLOB values.
+    **
+    ** Only do this on table-btrees so that writes to index-btrees do not
+    ** need to clear the cache.  This buys performance in the common case
+    ** in exchange for generality.
+    */
+    VdbeTxtBlbCache *pCache;
+    char *pBuf;
+    if( pC->colCache==0 ){
+      pC->pCache = sqlite3DbMallocZero(db, sizeof(VdbeTxtBlbCache) );
+      if( pC->pCache==0 ) return SQLITE_NOMEM;
+      pC->colCache = 1;
+    }
+    pCache = pC->pCache;
+    if( pCache->pCValue==0
+     || pCache->iCol!=iCol
+     || pCache->cacheStatus!=cacheStatus
+     || pCache->colCacheCtr!=colCacheCtr
+     || pCache->iOffset!=sqlite3BtreeOffset(pC->uc.pCursor)
+    ){
+      if( pCache->pCValue ) sqlite3RCStrUnref(pCache->pCValue);
+      pBuf = pCache->pCValue = sqlite3RCStrNew( len+3 );
+      if( pBuf==0 ) return SQLITE_NOMEM;
+      rc = sqlite3BtreePayload(pC->uc.pCursor, iOffset, len, pBuf);
+      if( rc ) return rc;
+      pBuf[len] = 0;
+      pBuf[len+1] = 0;
+      pBuf[len+2] = 0;
+      pCache->iCol = iCol;
+      pCache->cacheStatus = cacheStatus;
+      pCache->colCacheCtr = colCacheCtr;
+      pCache->iOffset = sqlite3BtreeOffset(pC->uc.pCursor);
+    }else{
+      pBuf = pCache->pCValue;
+    }
+    assert( t>=12 );
+    sqlite3RCStrRef(pBuf);
+    if( t&1 ){
+      rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding,
+                                (void(*)(void*))sqlite3RCStrUnref);
+      pDest->flags |= MEM_Term;
+    }else{
+      rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0,
+                                (void(*)(void*))sqlite3RCStrUnref);
+    }
+  }else{
+    rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest);
+    if( rc ) return rc;
+    sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
+    if( (t&1)!=0 && encoding==SQLITE_UTF8 ){
+      pDest->z[len] = 0;
+      pDest->flags |= MEM_Term;
+    }
+  }
+  pDest->flags &= ~MEM_Ephem;
+  return rc;
+}
+
+
 /*
 ** Return the symbolic name for the data type of a pMem
 */
@@ -737,6 +824,7 @@ int sqlite3VdbeExec(
   Mem *pIn2 = 0;             /* 2nd input operand */
   Mem *pIn3 = 0;             /* 3rd input operand */
   Mem *pOut = 0;             /* Output operand */
+  u32 colCacheCtr = 0;       /* Column cache counter */
 #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
   u64 *pnCycle = 0;
   int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0;
@@ -2272,10 +2360,10 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
 ** opcodes are allowed to occur between this instruction and the previous
 ** OP_Lt or OP_Gt.
 **
-** If result of an OP_Eq comparison on the same two operands as the
-** prior OP_Lt or OP_Gt would have been true, then jump to P2.
-** If the result of an OP_Eq comparison on the two previous
-** operands would have been false or NULL, then fall through.
+** If the result of an OP_Eq comparison on the same two operands as
+** the prior OP_Lt or OP_Gt would have been true, then jump to P2.  If
+** the result of an OP_Eq comparison on the two previous operands
+** would have been false or NULL, then fall through.
 */
 case OP_ElseEq: {       /* same as TK_ESCAPE, jump */
 
@@ -2705,7 +2793,7 @@ case OP_IsType: {        /* jump */
 /* Opcode: ZeroOrNull P1 P2 P3 * *
 ** Synopsis: r[P2] = 0 OR NULL
 **
-** If all both registers P1 and P3 are NOT NULL, then store a zero in
+** If both registers P1 and P3 are NOT NULL, then store a zero in
 ** register P2.  If either registers P1 or P3 are NULL then put
 ** a NULL in register P2.
 */
@@ -3061,13 +3149,14 @@ op_column_restart:
   }else{
     u8 p5;
     pDest->enc = encoding;
+    assert( pDest->db==db );
     /* This branch happens only when content is on overflow pages */
     if( ((p5 = (pOp->p5 & OPFLAG_BYTELENARG))!=0
           && (p5==OPFLAG_TYPEOFARG
               || (t>=12 && ((t&1)==0 || p5==OPFLAG_BYTELENARG))
              )
         )
-     || (len = sqlite3VdbeSerialTypeLen(t))==0
+     || sqlite3VdbeSerialTypeLen(t)==0
     ){
       /* Content is irrelevant for
       **    1. the typeof() function,
@@ -3084,14 +3173,13 @@ op_column_restart:
       */
       sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest);
     }else{
-      if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big;
-      rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest);
-      if( rc!=SQLITE_OK ) goto abort_due_to_error;
-      sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
-      if( (t&1)!=0 && encoding==SQLITE_UTF8 ){
-        pDest->flags |= MEM_Term;
+      rc = vdbeColumnFromOverflow(pC, p2, t, aOffset[p2],
+                p->cacheCtr, colCacheCtr, pDest);
+      if( rc ){
+        if( rc==SQLITE_NOMEM ) goto no_mem;
+        if( rc==SQLITE_TOOBIG ) goto too_big;
+        goto abort_due_to_error;
       }
-      pDest->flags &= ~MEM_Ephem;
     }
   }
 
@@ -5106,13 +5194,13 @@ case OP_IfNotOpen: {        /* jump */
 ** operands to OP_NotFound and OP_IdxGT.
 **
 ** This opcode is an optimization attempt only.  If this opcode always
-** falls through, the correct answer is still obtained, but extra works
+** falls through, the correct answer is still obtained, but extra work
 ** is performed.
 **
 ** A value of N in the seekHit flag of cursor P1 means that there exists
 ** a key P3:N that will match some record in the index.  We want to know
 ** if it is possible for a record P3:P4 to match some record in the
-** index.  If it is not possible, we can skips some work.  So if seekHit
+** index.  If it is not possible, we can skip some work.  So if seekHit
 ** is less than P4, attempt to find out if a match is possible by running
 ** OP_NotFound.
 **
@@ -5624,6 +5712,7 @@ case OP_Insert: {
   );
   pC->deferredMoveto = 0;
   pC->cacheStatus = CACHE_STALE;
+  colCacheCtr++;
 
   /* Invoke the update-hook if required. */
   if( rc ) goto abort_due_to_error;
@@ -5677,10 +5766,10 @@ case OP_RowCell: {
 ** left in an undefined state.
 **
 ** If the OPFLAG_AUXDELETE bit is set on P5, that indicates that this
-** delete one of several associated with deleting a table row and all its
-** associated index entries.  Exactly one of those deletes is the "primary"
-** delete.  The others are all on OPFLAG_FORDELETE cursors or else are
-** marked with the AUXDELETE flag.
+** delete is one of several associated with deleting a table row and
+** all its associated index entries.  Exactly one of those deletes is
+** the "primary" delete.  The others are all on OPFLAG_FORDELETE
+** cursors or else are marked with the AUXDELETE flag.
 **
 ** If the OPFLAG_NCHANGE flag of P2 (NB: P2 not P5) is set, then the row
 ** change count is incremented (otherwise not).
@@ -5784,6 +5873,7 @@ case OP_Delete: {
 
   rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5);
   pC->cacheStatus = CACHE_STALE;
+  colCacheCtr++;
   pC->seekResult = 0;
   if( rc ) goto abort_due_to_error;
 
@@ -5851,7 +5941,7 @@ case OP_SorterCompare: {
 ** Write into register P2 the current sorter data for sorter cursor P1.
 ** Then clear the column header cache on cursor P3.
 **
-** This opcode is normally use to move a record out of the sorter and into
+** This opcode is normally used to move a record out of the sorter and into
 ** a register that is the source for a pseudo-table cursor created using
 ** OpenPseudo.  That pseudo-table cursor is the one that is identified by
 ** parameter P3.  Clearing the P3 column cache as part of this opcode saves
@@ -6660,7 +6750,7 @@ case OP_IdxGE:  {       /* jump, ncycle */
 ** file is given by P1.
 **
 ** The table being destroyed is in the main database file if P3==0.  If
-** P3==1 then the table to be clear is in the auxiliary database file
+** P3==1 then the table to be destroyed is in the auxiliary database file
 ** that is used to store tables create using CREATE TEMPORARY TABLE.
 **
 ** If AUTOVACUUM is enabled then it is possible that another root page
@@ -6720,8 +6810,8 @@ case OP_Destroy: {     /* out2 */
 ** in the database file is given by P1.  But, unlike Destroy, do not
 ** remove the table or index from the database file.
 **
-** The table being clear is in the main database file if P2==0.  If
-** P2==1 then the table to be clear is in the auxiliary database file
+** The table being cleared is in the main database file if P2==0.  If
+** P2==1 then the table to be cleared is in the auxiliary database file
 ** that is used to store tables create using CREATE TEMPORARY TABLE.
 **
 ** If the P3 value is non-zero, then the row change count is incremented
@@ -7713,6 +7803,7 @@ case OP_JournalMode: {    /* out2 */
        || eNew==PAGER_JOURNALMODE_OFF
        || eNew==PAGER_JOURNALMODE_MEMORY
        || eNew==PAGER_JOURNALMODE_WAL
+       || eNew==PAGER_JOURNALMODE_WAL2
        || eNew==PAGER_JOURNALMODE_QUERY
   );
   assert( pOp->p1>=0 && pOp->p1<db->nDb );
@@ -7731,16 +7822,25 @@ case OP_JournalMode: {    /* out2 */
   /* Do not allow a transition to journal_mode=WAL for a database
   ** in temporary storage or if the VFS does not support shared memory
   */
-  if( eNew==PAGER_JOURNALMODE_WAL
+  if( isWalMode(eNew)
    && (sqlite3Strlen30(zFilename)==0           /* Temp file */
        || !sqlite3PagerWalSupported(pPager))   /* No shared-memory support */
   ){
     eNew = eOld;
   }
 
-  if( (eNew!=eOld)
-   && (eOld==PAGER_JOURNALMODE_WAL || eNew==PAGER_JOURNALMODE_WAL)
-  ){
+  if( eNew!=eOld && (isWalMode(eNew) || isWalMode(eOld)) ){
+
+    /* Prevent changing directly to wal2 from wal mode. And vice versa. */
+    if( isWalMode(eNew) && isWalMode(eOld) ){
+      rc = SQLITE_ERROR;
+      sqlite3VdbeError(p, "cannot change from %s to %s mode",
+          sqlite3JournalModename(eOld), sqlite3JournalModename(eNew)
+      );
+      goto abort_due_to_error;
+    }
+
+    /* Prevent switching into or out of wal/wal2 mode mid-transaction */
     if( !db->autoCommit || db->nVdbeRead>1 ){
       rc = SQLITE_ERROR;
       sqlite3VdbeError(p,
@@ -7748,31 +7848,34 @@ case OP_JournalMode: {    /* out2 */
           (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of")
       );
       goto abort_due_to_error;
-    }else{
-
-      if( eOld==PAGER_JOURNALMODE_WAL ){
-        /* If leaving WAL mode, close the log file. If successful, the call
-        ** to PagerCloseWal() checkpoints and deletes the write-ahead-log
-        ** file. An EXCLUSIVE lock may still be held on the database file
-        ** after a successful return.
-        */
-        rc = sqlite3PagerCloseWal(pPager, db);
-        if( rc==SQLITE_OK ){
-          sqlite3PagerSetJournalMode(pPager, eNew);
-        }
-      }else if( eOld==PAGER_JOURNALMODE_MEMORY ){
-        /* Cannot transition directly from MEMORY to WAL.  Use mode OFF
-        ** as an intermediate */
-        sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF);
-      }
+    }
  
-      /* Open a transaction on the database file. Regardless of the journal
-      ** mode, this transaction always uses a rollback journal.
+    if( isWalMode(eOld) ){
+      /* If leaving WAL mode, close the log file. If successful, the call
+      ** to PagerCloseWal() checkpoints and deletes the write-ahead-log 
+      ** file. An EXCLUSIVE lock may still be held on the database file 
+      ** after a successful return. 
       */
-      assert( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE );
+      rc = sqlite3PagerCloseWal(pPager, db);
       if( rc==SQLITE_OK ){
-        rc = sqlite3BtreeSetVersion(pBt, (eNew==PAGER_JOURNALMODE_WAL ? 2 : 1));
+        sqlite3PagerSetJournalMode(pPager, eNew);
       }
+    }else if( eOld==PAGER_JOURNALMODE_MEMORY ){
+      /* Cannot transition directly from MEMORY to WAL.  Use mode OFF
+      ** as an intermediate */
+      sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF);
+    }
+
+    /* Open a transaction on the database file. Regardless of the journal
+    ** mode, this transaction always uses a rollback journal.
+    */
+    assert( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE );
+    if( rc==SQLITE_OK ){
+      /* 1==rollback, 2==wal, 3==wal2 */
+      rc = sqlite3BtreeSetVersion(pBt, 
+          1 + isWalMode(eNew) + (eNew==PAGER_JOURNALMODE_WAL2)
+      );
+
     }
   }
 #endif /* ifndef SQLITE_OMIT_WAL */

src/vdbeInt.h

@@ -56,6 +56,9 @@ typedef struct VdbeSorter VdbeSorter;
 /* Elements of the linked list at Vdbe.pAuxData */
 typedef struct AuxData AuxData;
 
+/* A cache of large TEXT or BLOB values in a VdbeCursor */
+typedef struct VdbeTxtBlbCache VdbeTxtBlbCache;
+
 /* Types of VDBE cursors */
 #define CURTYPE_BTREE       0
 #define CURTYPE_SORTER      1
@@ -87,6 +90,7 @@ struct VdbeCursor {
   Bool useRandomRowid:1;  /* Generate new record numbers semi-randomly */
   Bool isOrdered:1;       /* True if the table is not BTREE_UNORDERED */
   Bool noReuse:1;         /* OpenEphemeral may not reuse this cursor */
+  Bool colCache:1;        /* pCache pointer is initialized and non-NULL */
   u16 seekHit;            /* See the OP_SeekHit and OP_IfNoHope opcodes */
   union {                 /* pBtx for isEphermeral.  pAltMap otherwise */
     Btree *pBtx;            /* Separate file holding temporary table */
@@ -127,6 +131,7 @@ struct VdbeCursor {
 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
   u64 maskUsed;           /* Mask of columns used by this cursor */
 #endif
+  VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */
 
   /* 2*nField extra array elements allocated for aType[], beyond the one
   ** static element declared in the structure.  nField total array slots for
@@ -139,12 +144,25 @@ struct VdbeCursor {
 #define IsNullCursor(P) \
   ((P)->eCurType==CURTYPE_PSEUDO && (P)->nullRow && (P)->seekResult==0)
 
-
 /*
 ** A value for VdbeCursor.cacheStatus that means the cache is always invalid.
 */
 #define CACHE_STALE 0
 
+/*
+** Large TEXT or BLOB values can be slow to load, so we want to avoid
+** loading them more than once.  For that reason, large TEXT and BLOB values
+** can be stored in a cache defined by this object, and attached to the
+** VdbeCursor using the pCache field.
+*/
+struct VdbeTxtBlbCache {
+  char *pCValue;        /* A RCStr buffer to hold the value */
+  i64 iOffset;          /* File offset of the row being cached */
+  int iCol;             /* Column for which the cache is valid */
+  u32 cacheStatus;      /* Vdbe.cacheCtr value */
+  u32 colCacheCtr;      /* Column cache counter */
+};
+
 /*
 ** When a sub-program is executed (OP_Program), a structure of this type
 ** is allocated to store the current value of the program counter, as
@@ -465,16 +483,18 @@ struct Vdbe {
   u32 nWrite;             /* Number of write operations that have occurred */
 #endif
   u16 nResColumn;         /* Number of columns in one row of the result set */
+  u16 nResAlloc;          /* Column slots allocated to aColName[] */
   u8 errorAction;         /* Recovery action to do in case of an error */
   u8 minWriteFileFormat;  /* Minimum file format for writable database files */
   u8 prepFlags;           /* SQLITE_PREPARE_* flags */
   u8 eVdbeState;          /* On of the VDBE_*_STATE values */
   bft expired:2;          /* 1: recompile VM immediately  2: when convenient */
-  bft explain:2;          /* True if EXPLAIN present on SQL command */
+  bft explain:2;          /* 0: normal, 1: EXPLAIN, 2: EXPLAIN QUERY PLAN */
   bft changeCntOn:1;      /* True to update the change-counter */
   bft usesStmtJournal:1;  /* True if uses a statement journal */
   bft readOnly:1;         /* True for statements that do not write */
   bft bIsReader:1;        /* True for statements that read */
+  bft haveEqpOps:1;       /* Bytecode supports EXPLAIN QUERY PLAN */
   yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
   yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
   u32 aCounter[9];        /* Counters used by sqlite3_stmt_status() */

src/vdbeapi.c

@@ -1127,7 +1127,8 @@ int sqlite3_aggregate_count(sqlite3_context *p){
 */
 int sqlite3_column_count(sqlite3_stmt *pStmt){
   Vdbe *pVm = (Vdbe *)pStmt;
-  return pVm ? pVm->nResColumn : 0;
+  if( pVm==0 ) return 0;
+  return pVm->nResColumn;
 }
 
 /*
@@ -1300,6 +1301,32 @@ int sqlite3_column_type(sqlite3_stmt *pStmt, int i){
   return iType;
 }
 
+/*
+** Column names appropriate for EXPLAIN or EXPLAIN QUERY PLAN.
+*/
+static const char * const azExplainColNames8[] = {
+   "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment",  /* EXPLAIN */
+   "id", "parent", "notused", "detail"                         /* EQP */
+};
+static const u16 azExplainColNames16data[] = {
+  /*   0 */  'a', 'd', 'd', 'r',                0,
+  /*   5 */  'o', 'p', 'c', 'o', 'd', 'e',      0,
+  /*  12 */  'p', '1',                          0, 
+  /*  15 */  'p', '2',                          0,
+  /*  18 */  'p', '3',                          0,
+  /*  21 */  'p', '4',                          0,
+  /*  24 */  'p', '5',                          0,
+  /*  27 */  'c', 'o', 'm', 'm', 'e', 'n', 't', 0,
+  /*  35 */  'i', 'd',                          0,
+  /*  38 */  'p', 'a', 'r', 'e', 'n', 't',      0,
+  /*  45 */  'n', 'o', 't', 'u', 's', 'e', 'd', 0,
+  /*  53 */  'd', 'e', 't', 'a', 'i', 'l',      0
+};
+static const u8 iExplainColNames16[] = {
+  0, 5, 12, 15, 18, 21, 24, 27,
+  35, 38, 45, 53
+};
+
 /*
 ** Convert the N-th element of pStmt->pColName[] into a string using
 ** xFunc() then return that string.  If N is out of range, return 0.
@@ -1332,15 +1359,29 @@ static const void *columnName(
     return 0;
   }
 #endif
+  if( N<0 ) return 0;
   ret = 0;
   p = (Vdbe *)pStmt;
   db = p->db;
   assert( db!=0 );
-  n = sqlite3_column_count(pStmt);
-  if( N<n && N>=0 ){
+  sqlite3_mutex_enter(db->mutex);
+
+  if( p->explain ){
+    if( useType>0 ) goto columnName_end;
+    n = p->explain==1 ? 8 : 4;
+    if( N>=n ) goto columnName_end;
+    if( useUtf16 ){
+      int i = iExplainColNames16[N + 8*p->explain - 8];
+      ret = (void*)&azExplainColNames16data[i];
+    }else{
+      ret = (void*)azExplainColNames8[N + 8*p->explain - 8];
+    }
+    goto columnName_end;
+  }
+  n = p->nResColumn;
+  if( N<n ){
     u8 prior_mallocFailed = db->mallocFailed;
     N += useType*n;
-    sqlite3_mutex_enter(db->mutex);
 #ifndef SQLITE_OMIT_UTF16
     if( useUtf16 ){
       ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]);
@@ -1357,8 +1398,9 @@ static const void *columnName(
       sqlite3OomClear(db);
       ret = 0;
     }
-    sqlite3_mutex_leave(db->mutex);
   }
+columnName_end:
+  sqlite3_mutex_leave(db->mutex);
   return ret;
 }
 
@@ -1815,6 +1857,39 @@ int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){
   return pStmt ? ((Vdbe*)pStmt)->explain : 0;
 }
 
+/*
+** Set the explain mode for a statement.
+*/
+int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode){
+  Vdbe *v = (Vdbe*)pStmt;
+  int rc;
+  sqlite3_mutex_enter(v->db->mutex);
+  if( v->explain==eMode ){
+    rc = SQLITE_OK;
+  }else if( eMode<0 || eMode>2 ){
+    rc = SQLITE_ERROR;
+  }else if( (v->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){
+    rc = SQLITE_ERROR;
+  }else if( v->eVdbeState!=VDBE_READY_STATE ){
+    rc = SQLITE_BUSY;
+  }else if( v->nMem>=10 && (eMode!=2 || v->haveEqpOps) ){
+    /* No reprepare necessary */
+    v->explain = eMode;
+    rc = SQLITE_OK;
+  }else{
+    v->explain = eMode;
+    rc = sqlite3Reprepare(v);
+    v->haveEqpOps = eMode==2;
+  }
+  if( v->explain ){
+    v->nResColumn = 12 - 4*v->explain;
+  }else{
+    v->nResColumn = v->nResAlloc;
+  }
+  sqlite3_mutex_leave(v->db->mutex);
+  return rc;
+}
+
 /*
 ** Return true if the prepared statement is in need of being reset.
 */

src/vdbeaux.c

@@ -1498,7 +1498,6 @@ void sqlite3VdbeReleaseRegisters(
 }
 #endif /* SQLITE_DEBUG */
 
-
 /*
 ** Change the value of the P4 operand for a specific instruction.
 ** This routine is useful when a large program is loaded from a
@@ -2418,8 +2417,8 @@ int sqlite3VdbeList(
         sqlite3VdbeMemSetInt64(pMem, pOp->p1);
         sqlite3VdbeMemSetInt64(pMem+1, pOp->p2);
         sqlite3VdbeMemSetInt64(pMem+2, pOp->p3);
-        sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free);    
-        p->nResColumn = 4;
+        sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free);
+        assert( p->nResColumn==4 );
       }else{
         sqlite3VdbeMemSetInt64(pMem+0, i);
         sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode),
@@ -2438,7 +2437,7 @@ int sqlite3VdbeList(
         sqlite3VdbeMemSetNull(pMem+7);
 #endif
         sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free);
-        p->nResColumn = 8;
+        assert( p->nResColumn==8 );
       }
       p->pResultRow = pMem;
       if( db->mallocFailed ){
@@ -2652,26 +2651,9 @@ void sqlite3VdbeMakeReady(
   resolveP2Values(p, &nArg);
   p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort);
   if( pParse->explain ){
-    static const char * const azColName[] = {
-       "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment",
-       "id", "parent", "notused", "detail"
-    };
-    int iFirst, mx, i;
     if( nMem<10 ) nMem = 10;
     p->explain = pParse->explain;
-    if( pParse->explain==2 ){
-      sqlite3VdbeSetNumCols(p, 4);
-      iFirst = 8;
-      mx = 12;
-    }else{
-      sqlite3VdbeSetNumCols(p, 8);
-      iFirst = 0;
-      mx = 8;
-    }
-    for(i=iFirst; i<mx; i++){
-      sqlite3VdbeSetColName(p, i-iFirst, COLNAME_NAME,
-                            azColName[i], SQLITE_STATIC);
-    }
+    p->nResColumn = 12 - 4*p->explain;
   }
   p->expired = 0;
 
@@ -2723,7 +2705,23 @@ void sqlite3VdbeMakeReady(
 void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){
   if( pCx ) sqlite3VdbeFreeCursorNN(p,pCx);
 }
+static SQLITE_NOINLINE void freeCursorWithCache(Vdbe *p, VdbeCursor *pCx){
+  VdbeTxtBlbCache *pCache = pCx->pCache;
+  assert( pCx->colCache );
+  pCx->colCache = 0;
+  pCx->pCache = 0;
+  if( pCache->pCValue ){
+    sqlite3RCStrUnref(pCache->pCValue);
+    pCache->pCValue = 0;
+  }
+  sqlite3DbFree(p->db, pCache);
+  sqlite3VdbeFreeCursorNN(p, pCx);
+}
 void sqlite3VdbeFreeCursorNN(Vdbe *p, VdbeCursor *pCx){
+  if( pCx->colCache ){
+    freeCursorWithCache(p, pCx);
+    return;
+  }
   switch( pCx->eCurType ){
     case CURTYPE_SORTER: {
       sqlite3VdbeSorterClose(p->db, pCx);
@@ -2824,12 +2822,12 @@ void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){
   int n;
   sqlite3 *db = p->db;
 
-  if( p->nResColumn ){
-    releaseMemArray(p->aColName, p->nResColumn*COLNAME_N);
+  if( p->nResAlloc ){
+    releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N);
     sqlite3DbFree(db, p->aColName);
   }
   n = nResColumn*COLNAME_N;
-  p->nResColumn = (u16)nResColumn;
+  p->nResColumn = p->nResAlloc = (u16)nResColumn;
   p->aColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n );
   if( p->aColName==0 ) return;
   initMemArray(p->aColName, n, db, MEM_Null);
@@ -2854,14 +2852,14 @@ int sqlite3VdbeSetColName(
 ){
   int rc;
   Mem *pColName;
-  assert( idx<p->nResColumn );
+  assert( idx<p->nResAlloc );
   assert( var<COLNAME_N );
   if( p->db->mallocFailed ){
     assert( !zName || xDel!=SQLITE_DYNAMIC );
     return SQLITE_NOMEM_BKPT;
   }
   assert( p->aColName!=0 );
-  pColName = &(p->aColName[idx+var*p->nResColumn]);
+  pColName = &(p->aColName[idx+var*p->nResAlloc]);
   rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel);
   assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 );
   return rc;
@@ -3374,6 +3372,7 @@ int sqlite3VdbeHalt(Vdbe *p){
           sqlite3VdbeLeave(p);
           return SQLITE_BUSY;
         }else if( rc!=SQLITE_OK ){
+          sqlite3SystemError(db, rc);
           p->rc = rc;
           sqlite3RollbackAll(db, SQLITE_OK);
           p->nChange = 0;
@@ -3685,7 +3684,7 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
   assert( db!=0 );
   assert( p->db==0 || p->db==db );
   if( p->aColName ){
-    releaseMemArray(p->aColName, p->nResColumn*COLNAME_N);
+    releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N);
     sqlite3DbNNFreeNN(db, p->aColName);
   }
   for(pSub=p->pProgram; pSub; pSub=pNext){

src/vdbemem.c

@@ -322,18 +322,26 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){
 ** this routine is a no-op.
 */
 void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){
-  if( (pMem->flags & (MEM_Str|MEM_Term))!=MEM_Str ) return;
+  if( (pMem->flags & (MEM_Str|MEM_Term|MEM_Ephem|MEM_Static))!=MEM_Str ){
+    /* pMem must be a string, and it cannot be an ephemeral or static string */
+    return;
+  }
   if( pMem->enc!=SQLITE_UTF8 ) return;
   if( NEVER(pMem->z==0) ) return;
   if( pMem->flags & MEM_Dyn ){
     if( pMem->xDel==sqlite3_free
      && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1)
-     && pMem->z[pMem->n]==0 
     ){
+      pMem->z[pMem->n] = 0;
       pMem->flags |= MEM_Term;
       return;
     }
-  }else if( pMem->szMalloc>0 && pMem->szMalloc >= pMem->n+1 ){
+    if( pMem->xDel==(void(*)(void*))sqlite3RCStrUnref ){
+      /* Blindly assume that all RCStr objects are zero-terminated */
+      pMem->flags |= MEM_Term;
+      return;
+    }
+  }else if( pMem->szMalloc >= pMem->n+1 ){
     pMem->z[pMem->n] = 0;
     pMem->flags |= MEM_Term;
     return;
@@ -1363,6 +1371,24 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){
   return valueToText(pVal, enc);
 }
 
+/* Return true if sqlit3_value object pVal is a string or blob value
+** that uses the destructor specified in the second argument.
+**
+** TODO:  Maybe someday promote this interface into a published API so
+** that third-party extensions can get access to it?
+*/
+int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){
+  if( ALWAYS(pVal!=0)
+   && ALWAYS((pVal->flags & (MEM_Str|MEM_Blob))!=0)
+   && (pVal->flags & MEM_Dyn)!=0
+   && pVal->xDel==xFree
+  ){
+    return 1;
+  }else{
+    return 0;
+  }
+}
+
 /*
 ** Create a new sqlite3_value object.
 */

src/vdbevtab.c

@@ -69,6 +69,8 @@ static int bytecodevtabConnect(
       "p5 INT,"
       "comment TEXT,"
       "subprog TEXT," 
+      "nexec INT,"
+      "ncycle INT,"
       "stmt HIDDEN"
     ");",
 
@@ -231,7 +233,7 @@ static int bytecodevtabColumn(
           }
         }
       }
-      i += 10;
+      i += 20;
     }
   }
   switch( i ){
@@ -281,16 +283,31 @@ static int bytecodevtabColumn(
       }
       break;
     }
-    case 10:  /* tables_used.type */
+
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+    case 9:     /* nexec */
+      sqlite3_result_int(ctx, pOp->nExec);
+      break;
+    case 10:    /* ncycle */
+      sqlite3_result_int(ctx, pOp->nCycle);
+      break;
+#else
+    case 9:     /* nexec */
+    case 10:    /* ncycle */
+      sqlite3_result_int(ctx, 0);
+      break;
+#endif
+
+    case 20:  /* tables_used.type */
       sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC);
       break;
-    case 11:  /* tables_used.schema */
+    case 21:  /* tables_used.schema */
       sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC);
       break;
-    case 12:  /* tables_used.name */
+    case 22:  /* tables_used.name */
       sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC);
       break;
-    case 13:  /* tables_used.wr */
+    case 23:  /* tables_used.wr */
       sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite);
       break;
   }
@@ -364,7 +381,7 @@ static int bytecodevtabBestIndex(
   int rc = SQLITE_CONSTRAINT;
   struct sqlite3_index_constraint *p;
   bytecodevtab *pVTab = (bytecodevtab*)tab;
-  int iBaseCol = pVTab->bTablesUsed ? 4 : 8;
+  int iBaseCol = pVTab->bTablesUsed ? 4 : 10;
   pIdxInfo->estimatedCost = (double)100;
   pIdxInfo->estimatedRows = 100;
   pIdxInfo->idxNum = 0;

src/wal.h

@@ -26,7 +26,7 @@
 #define CKPT_SYNC_FLAGS(X)  (((X)>>2)&0x03)
 
 #ifdef SQLITE_OMIT_WAL
-# define sqlite3WalOpen(x,y,z)                   0
+# define sqlite3WalOpen(w,x,y,z)                 0
 # define sqlite3WalLimit(x,y)
 # define sqlite3WalClose(v,w,x,y,z)              0
 # define sqlite3WalBeginReadTransaction(y,z)     0
@@ -45,6 +45,7 @@
 # define sqlite3WalFramesize(z)                  0
 # define sqlite3WalFindFrame(x,y,z)              0
 # define sqlite3WalFile(x)                       0
+# define sqlite3WalJournalMode(x)                0
 #else
 
 #define WAL_SAVEPOINT_NDATA 4
@@ -55,7 +56,7 @@
 typedef struct Wal Wal;
 
 /* Open and close a connection to a write-ahead log. */
-int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**);
+int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *,int,i64,int,Wal**);
 int sqlite3WalClose(Wal *pWal, sqlite3*, int sync_flags, int, u8 *);
 
 /* Set the limiting size of a WAL file. */
@@ -146,10 +147,17 @@ int sqlite3WalFramesize(Wal *pWal);
 /* Return the sqlite3_file object for the WAL file */
 sqlite3_file *sqlite3WalFile(Wal *pWal);
 
+/* Return the journal mode (WAL or WAL2) used by this Wal object. */
+int sqlite3WalJournalMode(Wal *pWal);
+
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
 int sqlite3WalWriteLock(Wal *pWal, int bLock);
 void sqlite3WalDb(Wal *pWal, sqlite3 *db);
 #endif
 
+#ifdef SQLITE_USE_SEH
+int sqlite3WalSystemErrno(Wal*);
+#endif
+
 #endif /* ifndef SQLITE_OMIT_WAL */
 #endif /* SQLITE_WAL_H */

src/where.c

@@ -2535,7 +2535,7 @@ static WhereLoop **whereLoopFindLesser(
 
     /* If pTemplate is always better than p, then cause p to be overwritten
     ** with pTemplate.  pTemplate is better than p if:
-    **   (1)  pTemplate has no more dependences than p, and
+    **   (1)  pTemplate has no more dependencies than p, and
     **   (2)  pTemplate has an equal or lower cost than p.
     */
     if( (p->prereq & pTemplate->prereq)==pTemplate->prereq   /* (1)  */
@@ -5003,7 +5003,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
   ** For joins of 3 or more tables, track the 10 best paths */
   mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10);
   assert( nLoop<=pWInfo->pTabList->nSrc );
-  WHERETRACE(0x002, ("---- begin solver.  (nRowEst=%d)\n", nRowEst));
+  WHERETRACE(0x002, ("---- begin solver.  (nRowEst=%d, nQueryLoop=%d)\n",
+                     nRowEst, pParse->nQueryLoop));
 
   /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
   ** case the purpose of this call is to estimate the number of rows returned
@@ -5757,6 +5758,28 @@ static SQLITE_NOINLINE void whereAddIndexedExpr(
   }
 }
 
+/*
+** Set the reverse-scan order mask to one for all tables in the query
+** with the exception of MATERIALIZED common table expressions that have
+** their own internal ORDER BY clauses.
+**
+** This implements the PRAGMA reverse_unordered_selects=ON setting.
+** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER).
+*/
+static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){
+  int ii;
+  for(ii=0; ii<pWInfo->pTabList->nSrc; ii++){
+    SrcItem *pItem = &pWInfo->pTabList->a[ii];
+    if( !pItem->fg.isCte
+     || pItem->u2.pCteUse->eM10d!=M10d_Yes
+     || NEVER(pItem->pSelect==0)
+     || pItem->pSelect->pOrderBy==0
+    ){
+      pWInfo->revMask |= MASKBIT(ii);
+    }
+  }
+}
+
 /*
 ** Generate the beginning of the loop used for WHERE clause processing.
 ** The return value is a pointer to an opaque structure that contains
@@ -6121,8 +6144,9 @@ WhereInfo *sqlite3WhereBegin(
        if( db->mallocFailed ) goto whereBeginError;
     }
   }
+  assert( pWInfo->pTabList!=0 );
   if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
-     pWInfo->revMask = ALLBITS;
+    whereReverseScanOrder(pWInfo);
   }
   if( pParse->nErr ){
     goto whereBeginError;
@@ -6222,6 +6246,7 @@ WhereInfo *sqlite3WhereBegin(
         0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW)
      && !IsVirtual(pTabList->a[0].pTab)
      && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK))
+     && OptimizationEnabled(db, SQLITE_OnePass)
     )){
       pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
       if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){

test/corruptA.test

@@ -47,7 +47,7 @@ db close
 forcecopy test.db test.db-template
 
 set unreadable_version 02
-ifcapable wal { set unreadable_version 03 }
+ifcapable wal { set unreadable_version 04 }
 do_test corruptA-2.1 {
   forcecopy test.db-template test.db
   hexio_write test.db 19 $unreadable_version   ;# the read format number

test/corruptL.test

@@ -1269,6 +1269,7 @@ do_test 15.0 {
 }]} {}
 
 extra_schema_checks 0
+optimization_control db one-pass off
 do_catchsql_test 15.1 {
   PRAGMA cell_size_check = 0;
   UPDATE c1 SET c= NOT EXISTS(SELECT 1 FROM c1 ORDER BY (SELECT 1 FROM c1 ORDER BY a)) +10 WHERE d BETWEEN 4 AND 7;

test/json/README.md

@@ -1,5 +1,6 @@
 The files in this subdirectory are used to help measure the performance
-of the SQLite JSON parser.
+of the SQLite JSON functions, especially in relation to handling large
+JSON inputs.
 
 # 1.0 Prerequisites
 
@@ -7,6 +8,8 @@ of the SQLite JSON parser.
 
   2.  Fossil
 
+  3.  tclsh
+
 # 2.0 Setup
 
   1.  Run: "`tclsh json-generator.tcl | sqlite3 json100mb.db`" to create
@@ -14,16 +17,17 @@ of the SQLite JSON parser.
       file lands in the directory from which you will run tests, not in
       the test/json subdirectory of the source tree.
 
-  2.  Build the baseline sqlite3.c file.  ("`make sqlite3.c`")
+  2.  Build the baseline sqlite3.c file with sqlite3.h and shell.c.
+      ("`CFLAGS='-Os -g' make -e clean sqlite3.c`")
 
-  3.  Run "`sh json-speed-check-1.sh trunk`".   This creates the baseline
+  3.  Run "`sh json-speed-check.sh trunk`".   This creates the baseline
       profile in "jout-trunk.txt".
 
 # 3.0 Testing
 
-  1.  Build the sqlite3.c to be tested.
+  1.  Build the sqlite3.c (with sqlite3.h and shell.c) to be tested.
 
-  2.  Run "`sh json-speed-check-1.sh x1`".  The profile output will appear
+  2.  Run "`sh json-speed-check.sh x1`".  The profile output will appear
       in jout-x1.txt.  Substitute any label you want in place of "x1".
 
   3.  Run the script shown below in the CLI.

test/json/json-q1.txt

@@ -2,3 +2,23 @@
 .timer on
 .param set $label 'q87'
 SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL;
+
+CREATE TEMP TABLE t2(x JSON TEXT);
+WITH RECURSIVE
+  c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<25000),
+  array1(y) AS (
+    SELECT json_group_array(
+             json_object('x',x,'y',random(),'z',hex(randomblob(50)))
+           )
+      FROM c
+  ),
+  c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<5)
+INSERT INTO t2(x)
+  SELECT json_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6) FROM array1, c2;
+CREATE INDEX t2x1 ON t2(x->>'a');
+CREATE INDEX t2x2 ON t2(x->>'b');
+CREATE INDEX t2x3 ON t2(x->>'e');
+CREATE INDEX t2x4 ON t2(x->>'f');
+UPDATE t2 SET x=json_replace(x,'$.f',(x->>'f')+1);
+UPDATE t2 SET x=json_set(x,'$.e',(x->>'f')-1);
+UPDATE t2 SET x=json_remove(x,'$.d');

test/json102.test

@@ -397,4 +397,22 @@ do_execsql_test json102-1610 {
   6 {}        null {}        null    {}        null
 ]
 
+reset_db
+do_execsql_test json102-1700 {
+  CREATE TABLE t1(a1 DATE, a2 INTEGER PRIMARY KEY, a3 INTEGER, memo TEXT);
+  CREATE INDEX t1x1 ON t1(a3, a1, memo->>'y');
+  INSERT INTO t1(a2,a1,a3,memo) VALUES (876, '2023-08-03', 5, '{"x":77,"y":4}');
+}
+do_execsql_test json102-1710 {
+  UPDATE t1 SET memo = JSON_REMOVE(memo, '$.y');
+  PRAGMA integrity_check;
+  SELECT * FROM t1;
+} {ok 2023-08-03 876 5 {{"x":77}}}
+do_execsql_test json102-1720 {
+  UPDATE t1 SET memo = JSON_SET(memo, '$.y', 6)
+    WHERE a2 IN (876) AND JSON_TYPE(memo, '$.y') IS NULL;
+  PRAGMA integrity_check;
+  SELECT * FROM t1;
+} {ok 2023-08-03 876 5 {{"x":77,"y":6}}}
+
 finish_test

test/permutations.test

@@ -462,18 +462,21 @@ lappend ::testsuitelist xxx
 test_suite "coverage-wal" -description {
   Coverage tests for file wal.c.
 } -files {
-  wal.test wal2.test wal3.test wal4.test wal5.test
+  wal2big.test wal2recover.test wal2rewrite.test
+  wal2simple.test wal2snapshot.test wal2.test
+  wal3.test wal4.test wal5.test
   wal64k.test wal6.test wal7.test wal8.test wal9.test
-  walbak.test walbig.test walblock.test walcksum.test walcrash2.test
-  walcrash3.test walcrash4.test walcrash.test walfault.test walhook.test
-  walmode.test walnoshm.test waloverwrite.test walpersist.test 
-  walprotocol2.test walprotocol.test walro2.test walrofault.test 
-  walro.test walshared.test walslow.test walvfs.test
-  walfault2.test
-  nockpt.test
-
+  walbak.test walbig.test walblock.test walcksum.test 
+  walfault.test walhook.test walmode.test walnoshm.test
+  waloverwrite.test walpersist.test walprotocol2.test
+  walprotocol.test walro2.test walrofault.test walro.test
+  walshared.test walslow.test wal.test
+  wal2savepoint.test wal2lock.test wal2recover2.test
+  walvfs.test walfault2.test nockpt.test
   snapshot2.test snapshot3.test snapshot4.test
   snapshot_fault.test snapshot.test snapshot_up.test
+  walcrash2.test walcrash3.test walcrash4.test walcrash.test
+  wal2fault.test
 } 
 
 test_suite "coverage-pager" -description {
@@ -1037,6 +1040,23 @@ test_suite "wal" -description {
   fts3c.test   fts3d.test   fts3e.test   fts3query.test 
 }
 
+test_suite "wal2" -description {
+  Run tests with journal_mode=WAL2
+} -initialize {
+  set ::G(savepoint6_iterations) 100
+} -shutdown {
+  unset -nocomplain ::G(savepoint6_iterations)
+} -files {
+  savepoint.test     savepoint2.test     savepoint6.test
+  trans.test         avtrans.test
+
+  fts3aa.test  fts3ab.test  fts3ac.test  fts3ad.test
+  fts3ae.test  fts3af.test  fts3ag.test  fts3ah.test
+  fts3ai.test  fts3aj.test  fts3ak.test  fts3al.test
+  fts3am.test  fts3an.test  fts3ao.test  fts3b.test
+  fts3c.test   fts3d.test   fts3e.test   fts3query.test 
+}
+
 test_suite "rtree" -description {
   All R-tree related tests. Provides coverage of source file rtree.c.
 } -files [glob -nocomplain $::testdir/../ext/rtree/*.test]

test/rdonly.test

@@ -41,7 +41,7 @@ do_test rdonly-1.1.1 {
   sqlite3_db_readonly db main
 } {0}
 
-# Changes the write version from 1 to 3.  Verify that the database
+# Changes the write version from 1 to 4.  Verify that the database
 # can be read but not written.
 #
 do_test rdonly-1.2 {
@@ -49,7 +49,7 @@ do_test rdonly-1.2 {
   hexio_get_int [hexio_read test.db 18 1]
 } 1
 do_test rdonly-1.3 {
-  hexio_write test.db 18 03
+  hexio_write test.db 18 04
   sqlite3 db test.db
   execsql {
     SELECT * FROM t1;
@@ -83,7 +83,7 @@ do_test rdonly-1.5 {
 # the database is read-only until after it is locked.
 #
 set ro_version 02
-ifcapable wal { set ro_version 03 }
+ifcapable wal { set ro_version 04 }
 do_test rdonly-1.6 {
   hexio_write test.db 18 $ro_version     ; # write-version
   hexio_write test.db 24 11223344        ; # change-counter

test/savepoint.test

@@ -30,6 +30,7 @@ do_test savepoint-1.1 {
     RELEASE sp1;
   }
 } {}
+wal_check_journal_mode savepoint-1.1
 do_test savepoint-1.2 {
   execsql {
     SAVEPOINT sp1;
@@ -807,7 +808,8 @@ do_test savepoint-11.6 {
 integrity_check savepoint-11.7
 do_test savepoint-11.8 {
   execsql { ROLLBACK }
-  execsql { PRAGMA wal_checkpoint }
+  db close
+  sqlite3 db test.db
   file size test.db
 } {8192}
 

test/savepoint6.test

@@ -15,6 +15,10 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
 proc sql {zSql} {
+  if {0 && $::debug_op} {
+    puts stderr "$zSql ;"
+    flush stderr
+  }
   uplevel db eval [list $zSql]
   #puts stderr "$zSql ;"
 }
@@ -67,11 +71,13 @@ proc x_to_y {x} {
 #   delete_rows XVALUES
 #
 proc savepoint {zName} {
+  if {$::debug_op} { puts stderr "savepoint $zName" ; flush stderr }
   catch { sql "SAVEPOINT $zName" }
   lappend ::lSavepoint [list $zName [array get ::aEntry]]
 }
 
 proc rollback {zName} {
+  if {$::debug_op} { puts stderr "rollback $zName" ; flush stderr }
   catch { sql "ROLLBACK TO $zName" }
   for {set i [expr {[llength $::lSavepoint]-1}]} {$i>=0} {incr i -1} {
     set zSavepoint [lindex $::lSavepoint $i 0]
@@ -89,6 +95,7 @@ proc rollback {zName} {
 }
 
 proc release {zName} {
+  if {$::debug_op} { puts stderr "release $zName" ; flush stderr }
   catch { sql "RELEASE $zName" }
   for {set i [expr {[llength $::lSavepoint]-1}]} {$i>=0} {incr i -1} {
     set zSavepoint [lindex $::lSavepoint $i 0]
@@ -104,6 +111,7 @@ proc release {zName} {
 }
 
 proc insert_rows {lX} {
+  if {$::debug_op} { puts stderr "insert_rows $lX" ; flush stderr }
   foreach x $lX {
     set y [x_to_y $x]
 
@@ -116,6 +124,7 @@ proc insert_rows {lX} {
 }
 
 proc delete_rows {lX} {
+  if {$::debug_op} { puts stderr "delete_rows $lX" ; flush stderr }
   foreach x $lX {
     # Update database [db]
     sql "DELETE FROM t1 WHERE x = $x"
@@ -164,6 +173,11 @@ proc random_integers {nRes nRange} {
 } 
 #-------------------------------------------------------------------------
 
+set ::debug_op 0
+proc debug_ops {} {
+  set ::debug_op 1
+}
+
 proc database_op {} {
   set i [expr int(rand()*2)] 
   if {$i==0} {
@@ -185,9 +199,6 @@ proc savepoint_op {} {
   set C [lindex $cmds [expr int(rand()*6)]]
   set N [lindex $names [expr int(rand()*5)]]
 
-  #puts stderr "   $C $N ;  "
-  #flush stderr
-
   $C $N
   return ok
 }

test/snapshot_fault.test

@@ -23,6 +23,7 @@ set testprefix snapshot_fault
 # checkpointing the db.
 #
 do_faultsim_test 1.0 -prep {
+  catch { db2 close }
   faultsim_delete_and_reopen
   sqlite3 db2 test.db
   db2 eval { 

test/tester.tcl

@@ -613,6 +613,7 @@ proc reset_db {} {
   forcedelete test.db
   forcedelete test.db-journal
   forcedelete test.db-wal
+  forcedelete test.db-wal2
   sqlite3 db ./test.db
   set ::DB [sqlite3_connection_pointer db]
   if {[info exists ::SETUP_SQL]} {
@@ -2317,17 +2318,32 @@ proc drop_all_indexes {{db db}} {
 #     Returns true if this test should be run in WAL mode. False otherwise.
 #
 proc wal_is_wal_mode {} {
-  expr {[permutation] eq "wal"}
+  if {[permutation] eq "wal"} { return 1 }
+  if {[permutation] eq "wal2"} { return 2 }
+  return 0
 }
 proc wal_set_journal_mode {{db db}} {
-  if { [wal_is_wal_mode] } {
-    $db eval "PRAGMA journal_mode = WAL"
+  switch -- [wal_is_wal_mode] {
+    0 {
+    }
+
+    1 {
+      $db eval "PRAGMA journal_mode = WAL"
+    }
+
+    2 {
+      $db eval "PRAGMA journal_mode = WAL2"
+    }
   }
 }
 proc wal_check_journal_mode {testname {db db}} {
   if { [wal_is_wal_mode] } {
     $db eval { SELECT * FROM sqlite_master }
-    do_test $testname [list $db eval "PRAGMA main.journal_mode"] {wal}
+    set expected "wal"
+    if {[wal_is_wal_mode]==2} {
+      set expected "wal2"
+    }
+    do_test $testname [list $db eval "PRAGMA main.journal_mode"] $expected
   }
 }
 

test/update.test

@@ -764,5 +764,27 @@ do_execsql_test update-21.4 {
   SELECT * FROM t1 ORDER BY vkey, c5;
   ROLLBACK;
 } {6 -54 100 NULL}
+# Follow-up on 2023-07-31 (forum post https://sqlite.org/forum/forumpost/8ab195fd44e75ed0):
+# Only disable one-pass if the subquery is in the WHERE clause.  The SET expressions
+# do not count.
+do_execsql_test update-21.11 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT);
+  CREATE TABLE t2(d INT);
+}
+do_eqp_test update-21.12 {
+  WITH t3(x,y) AS (SELECT d, row_number()OVER() FROM t2)
+    UPDATE t1 SET b=(SELECT y FROM t3 WHERE t1.a=t3.x);
+} {
+  QUERY PLAN
+  |--SCAN t1
+  `--CORRELATED SCALAR SUBQUERY xxxxxx
+     |--CO-ROUTINE t3
+     |  |--CO-ROUTINE (subquery-xxxxxx)
+     |  |  `--SCAN t2
+     |  `--SCAN (subquery-xxxxxx)
+     |--BLOOM FILTER ON t3 (x=?)
+     `--SEARCH t3 USING AUTOMATIC COVERING INDEX (x=?)
+}
 
 finish_test

test/uri.test

@@ -282,11 +282,11 @@ ifcapable wal {
       INSERT INTO t2 VALUES('x', 'y');
     }
     lsort [array names ::T1]
-  } {test.db1 test.db1-journal test.db1-wal}
+  } {test.db1 test.db1-journal test.db1-wal test.db1-wal2}
   
   do_test 5.1.2 {
     lsort [array names ::T2]
-  } {test.db2 test.db2-journal test.db2-wal}
+  } {test.db2 test.db2-journal test.db2-wal test.db2-wal2}
   db close
   
   tvfs1 delete

test/wal.test

@@ -1175,7 +1175,7 @@ foreach {tn pgsz works} {
   9  32768    1
  10  65536    1
  11 131072    0
- 11   1016    0
+ 12   1016    0
 } {
 
   if {$::SQLITE_MAX_PAGE_SIZE < $pgsz} {
@@ -1185,14 +1185,14 @@ foreach {tn pgsz works} {
   for {set pg 1} {$pg <= 3} {incr pg} {
     forcecopy testX.db test.db
     forcedelete test.db-wal
-  
+
     # Check that the database now exists and consists of three pages. And
     # that there is no associated wal file.
     #
     do_test wal-18.2.$tn.$pg.1 { file exists test.db-wal } 0
     do_test wal-18.2.$tn.$pg.2 { file exists test.db } 1
     do_test wal-18.2.$tn.$pg.3 { file size test.db } [expr 1024*3]
-  
+
     do_test wal-18.2.$tn.$pg.4 {
 
       # Create a wal file that contains a single frame (database page
@@ -1224,16 +1224,16 @@ foreach {tn pgsz works} {
       puts -nonewline $fd $framehdr
       puts -nonewline $fd $framebody
       close $fd
-  
+
       file size test.db-wal
     } [wal_file_size 1 $pgsz]
-  
+
     do_test wal-18.2.$tn.$pg.5 {
       sqlite3 db test.db
       set rc [catch { db one {PRAGMA integrity_check} } msg]
       expr { $rc!=0 || $msg!="ok" }
     } $works
-  
+ 
     db close
   }
 }