Add support for the comma (,) modifier to %f formats in the format() function.

FossilOrigin-Name: 7080e196a1f887640ff51ddc508ec6796ce12874c2944855702753b64a8e5e50

ext/fts5/fts5Int.h

@@ -199,6 +199,7 @@ struct Fts5Config {
   int ePattern;                   /* FTS_PATTERN_XXX constant */
 
   /* Values loaded from the %_config table */
+  int iVersion;                   /* fts5 file format 'version' */
   int iCookie;                    /* Incremented when %_config is modified */
   int pgsz;                       /* Approximate page size used in %_data */
   int nAutomerge;                 /* 'automerge' setting */
@@ -207,6 +208,7 @@ struct Fts5Config {
   int nHashSize;                  /* Bytes of memory for in-memory hash */
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
+  int bSecureDelete;              /* 'secure-delete' */
 
   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   char **pzErrmsg;
@@ -216,8 +218,11 @@ struct Fts5Config {
 #endif
 };
 
-/* Current expected value of %_config table 'version' field */
-#define FTS5_CURRENT_VERSION  4
+/* Current expected value of %_config table 'version' field. And
+** the expected version if the 'secure-delete' option has ever been
+** set on the table.  */
+#define FTS5_CURRENT_VERSION               4
+#define FTS5_CURRENT_VERSION_SECUREDELETE  5
 
 #define FTS5_CONTENT_NORMAL   0
 #define FTS5_CONTENT_NONE     1
@@ -383,6 +388,7 @@ struct Fts5IndexIter {
 ** above. */
 #define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
 #define FTS5INDEX_QUERY_NOOUTPUT   0x0020
+#define FTS5INDEX_QUERY_SKIPHASH   0x0040
 
 /*
 ** Create/destroy an Fts5Index object.
@@ -537,7 +543,7 @@ int sqlite3Fts5GetVarintLen(u32 iVal);
 u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
 int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
 
-#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
+#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b))
 #define fts5GetVarint    sqlite3Fts5GetVarint
 
 #define fts5FastGetVarint32(a, iOff, nVal) {      \

ext/fts5/fts5_config.c

@@ -550,6 +550,7 @@ int sqlite3Fts5ConfigParse(
     rc = SQLITE_ERROR;
   }
 
+  assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK );
   for(i=3; rc==SQLITE_OK && i<nArg; i++){
     const char *zOrig = azArg[i];
     const char *z;
@@ -903,6 +904,18 @@ int sqlite3Fts5ConfigSetValue(
       rc = SQLITE_OK;
       *pbBadkey = 1;
     }
+  }
+
+  else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){
+    int bVal = -1;
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      bVal = sqlite3_value_int(pVal);
+    }
+    if( bVal<0 ){
+      *pbBadkey = 1;
+    }else{
+      pConfig->bSecureDelete = (bVal ? 1 : 0);
+    }
   }else{
     *pbBadkey = 1;
   }
@@ -947,15 +960,20 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
     rc = sqlite3_finalize(p);
   }
   
-  if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){
+  if( rc==SQLITE_OK 
+   && iVersion!=FTS5_CURRENT_VERSION
+   && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE
+  ){
     rc = SQLITE_ERROR;
     if( pConfig->pzErrmsg ){
       assert( 0==*pConfig->pzErrmsg );
-      *pConfig->pzErrmsg = sqlite3_mprintf(
-          "invalid fts5 file format (found %d, expected %d) - run 'rebuild'",
-          iVersion, FTS5_CURRENT_VERSION
+      *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format "
+          "(found %d, expected %d or %d) - run 'rebuild'",
+          iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
       );
     }
+  }else{
+    pConfig->iVersion = iVersion;
   }
 
   if( rc==SQLITE_OK ){

ext/fts5/fts5_expr.c

@@ -407,7 +407,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
   Fts5Parse sParse;
   memset(&sParse, 0, sizeof(sParse));
 
-  if( *pp1 ){
+  if( *pp1 && p2 ){
     Fts5Expr *p1 = *pp1;
     int nPhrase = p1->nPhrase + p2->nPhrase;
 
@@ -432,7 +432,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
     }
     sqlite3_free(p2->apExprPhrase);
     sqlite3_free(p2);
-  }else{
+  }else if( p2 ){
     *pp1 = p2;
   }
 

ext/fts5/fts5_index.c

@@ -302,6 +302,8 @@ struct Fts5Index {
   sqlite3_stmt *pIdxSelect;
   int nRead;                      /* Total number of blocks read */
 
+  sqlite3_stmt *pDeleteFromIdx;
+
   sqlite3_stmt *pDataVersion;
   i64 iStructVersion;             /* data_version when pStruct read */
   Fts5Structure *pStruct;         /* Current db structure (or NULL) */
@@ -394,9 +396,6 @@ struct Fts5CResult {
 ** iLeafOffset:
 **   Byte offset within the current leaf that is the first byte of the 
 **   position list data (one byte passed the position-list size field).
-**   rowid field of the current entry. Usually this is the size field of the
-**   position list data. The exception is if the rowid for the current entry 
-**   is the last thing on the leaf page.
 **
 ** pLeaf:
 **   Buffer containing current leaf page data. Set to NULL at EOF.
@@ -955,6 +954,7 @@ static int fts5StructureDecode(
             rc = FTS5_CORRUPT;
             break;
           }
+          assert( pSeg!=0 );
           i += fts5GetVarint32(&pData[i], pSeg->iSegid);
           i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst);
           i += fts5GetVarint32(&pData[i], pSeg->pgnoLast);
@@ -985,6 +985,7 @@ static int fts5StructureDecode(
 */
 static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){
   fts5StructureMakeWritable(pRc, ppStruct);
+  assert( (ppStruct!=0 && (*ppStruct)!=0) || (*pRc)!=SQLITE_OK );
   if( *pRc==SQLITE_OK ){
     Fts5Structure *pStruct = *ppStruct;
     int nLevel = pStruct->nLevel;
@@ -1443,42 +1444,25 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){
     pLvl->bEof = 1;
   }else{
     u8 *a = pLvl->pData->p;
-    i64 iVal;
-    int iLimit;
-    int ii;
-    int nZero = 0;
 
-    /* Currently iOff points to the first byte of a varint. This block 
-    ** decrements iOff until it points to the first byte of the previous 
-    ** varint. Taking care not to read any memory locations that occur
-    ** before the buffer in memory.  */
-    iLimit = (iOff>9 ? iOff-9 : 0);
-    for(iOff--; iOff>iLimit; iOff--){
-      if( (a[iOff-1] & 0x80)==0 ) break;
-    }
+    pLvl->iOff = 0;
+    fts5DlidxLvlNext(pLvl);
+    while( 1 ){
+      int nZero = 0;
+      int ii = pLvl->iOff;
+      u64 delta = 0;
 
-    fts5GetVarint(&a[iOff], (u64*)&iVal);
-    pLvl->iRowid -= iVal;
-    pLvl->iLeafPgno--;
-
-    /* Skip backwards past any 0x00 varints. */
-    for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){
-      nZero++;
-    }
-    if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){
-      /* The byte immediately before the last 0x00 byte has the 0x80 bit
-      ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80
-      ** bytes before a[ii]. */
-      int bZero = 0;              /* True if last 0x00 counts */
-      if( (ii-8)>=pLvl->iFirstOff ){
-        int j;
-        for(j=1; j<=8 && (a[ii-j] & 0x80); j++);
-        bZero = (j>8);
+      while( a[ii]==0 ){
+        nZero++;
+        ii++;
       }
-      if( bZero==0 ) nZero--;
+      ii += sqlite3Fts5GetVarint(&a[ii], &delta);
+
+      if( ii>=iOff ) break;
+      pLvl->iLeafPgno += nZero+1;
+      pLvl->iRowid += delta;
+      pLvl->iOff = ii;
     }
-    pLvl->iLeafPgno -= nZero;
-    pLvl->iOff = iOff - nZero;
   }
 
   return pLvl->bEof;
@@ -1674,7 +1658,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
   i64 iOff = pIter->iLeafOffset;
 
   ASSERT_SZLEAF_OK(pIter->pLeaf);
-  if( iOff>=pIter->pLeaf->szLeaf ){
+  while( iOff>=pIter->pLeaf->szLeaf ){
     fts5SegIterNextPage(p, pIter);
     if( pIter->pLeaf==0 ){
       if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT;
@@ -1773,10 +1757,12 @@ static void fts5SegIterInit(
     fts5SegIterSetNext(p, pIter);
     pIter->pSeg = pSeg;
     pIter->iLeafPgno = pSeg->pgnoFirst-1;
-    fts5SegIterNextPage(p, pIter);
+    do {
+      fts5SegIterNextPage(p, pIter);
+    }while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 );
   }
 
-  if( p->rc==SQLITE_OK ){
+  if( p->rc==SQLITE_OK && pIter->pLeaf ){
     pIter->iLeafOffset = 4;
     assert( pIter->pLeaf!=0 );
     assert_nc( pIter->pLeaf->nn>4 );
@@ -1970,7 +1956,7 @@ static void fts5SegIterNext_None(
   iOff = pIter->iLeafOffset;
 
   /* Next entry is on the next page */
-  if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
+  while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
     fts5SegIterNextPage(p, pIter);
     if( p->rc || pIter->pLeaf==0 ) return;
     pIter->iRowid = 0;
@@ -2163,7 +2149,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
   Fts5Data *pLast = 0;
   int pgnoLast = 0;
 
-  if( pDlidx ){
+  if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){
     int iSegid = pIter->pSeg->iSegid;
     pgnoLast = fts5DlidxIterPgno(pDlidx);
     pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast));
@@ -2724,7 +2710,8 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
 
 /*
 ** Move the seg-iter so that it points to the first rowid on page iLeafPgno.
-** It is an error if leaf iLeafPgno does not exist or contains no rowids.
+** It is an error if leaf iLeafPgno does not exist. Unless the db is
+** a 'secure-delete' db, if it contains no rowids then this is also an error.
 */
 static void fts5SegIterGotoPage(
   Fts5Index *p,                   /* FTS5 backend object */
@@ -2739,21 +2726,23 @@ static void fts5SegIterGotoPage(
     fts5DataRelease(pIter->pNextLeaf);
     pIter->pNextLeaf = 0;
     pIter->iLeafPgno = iLeafPgno-1;
-    fts5SegIterNextPage(p, pIter);
-    assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno );
 
-    if( p->rc==SQLITE_OK && ALWAYS(pIter->pLeaf!=0) ){
+    while( p->rc==SQLITE_OK ){
       int iOff;
-      u8 *a = pIter->pLeaf->p;
-      int n = pIter->pLeaf->szLeaf;
-
+      fts5SegIterNextPage(p, pIter);
+      if( pIter->pLeaf==0 ) break;
       iOff = fts5LeafFirstRowidOff(pIter->pLeaf);
-      if( iOff<4 || iOff>=n ){
-        p->rc = FTS5_CORRUPT;
-      }else{
-        iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
-        pIter->iLeafOffset = iOff;
-        fts5SegIterLoadNPos(p, pIter);
+      if( iOff>0 ){
+        u8 *a = pIter->pLeaf->p;
+        int n = pIter->pLeaf->szLeaf;
+        if( iOff<4 || iOff>=n ){
+          p->rc = FTS5_CORRUPT;
+        }else{
+          iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
+          pIter->iLeafOffset = iOff;
+          fts5SegIterLoadNPos(p, pIter);
+        }
+        break;
       }
     }
   }
@@ -3468,7 +3457,7 @@ static void fts5MultiIterNew(
     if( iLevel<0 ){
       assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
       nSeg = pStruct->nSegment;
-      nSeg += (p->pHash ? 1 : 0);
+      nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH));
     }else{
       nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
     }
@@ -3489,7 +3478,7 @@ static void fts5MultiIterNew(
   if( p->rc==SQLITE_OK ){
     if( iLevel<0 ){
       Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
-      if( p->pHash ){
+      if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){
         /* Add a segment iterator for the current contents of the hash table. */
         Fts5SegIter *pIter = &pNew->aSeg[iIter++];
         fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
@@ -4244,7 +4233,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
           fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr);
           fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n);
           fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p);
-          fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff,&pData->p[iOff]);
+          fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]);
           if( p->rc==SQLITE_OK ){
             /* Set the szLeaf field */
             fts5PutU16(&buf.p[2], (u16)buf.n);
@@ -4522,16 +4511,16 @@ static void fts5IndexCrisismerge(
 ){
   const int nCrisis = p->pConfig->nCrisisMerge;
   Fts5Structure *pStruct = *ppStruct;
-  int iLvl = 0;
-
-  assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 );
-  while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){
-    fts5IndexMergeLevel(p, &pStruct, iLvl, 0);
-    assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) );
-    fts5StructurePromote(p, iLvl+1, pStruct);
-    iLvl++;
+  if( pStruct && pStruct->nLevel>0 ){
+    int iLvl = 0;
+    while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){
+      fts5IndexMergeLevel(p, &pStruct, iLvl, 0);
+      assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) );
+      fts5StructurePromote(p, iLvl+1, pStruct);
+      iLvl++;
+    }
+    *ppStruct = pStruct;
   }
-  *ppStruct = pStruct;
 }
 
 static int fts5IndexReturn(Fts5Index *p){
@@ -4565,6 +4554,413 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){
   return ret;
 }
 
+/*
+** Execute the SQL statement:
+**
+**    DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno);
+**
+** This is used when a secure-delete operation removes the last term
+** from a segment leaf page. In that case the %_idx entry is removed 
+** too. This is done to ensure that if all instances of a token are
+** removed from an fts5 database in secure-delete mode, no trace of
+** the token itself remains in the database.
+*/
+static void fts5SecureDeleteIdxEntry(
+  Fts5Index *p,                   /* FTS5 backend object */
+  int iSegid,                     /* Id of segment to delete entry for */
+  int iPgno                       /* Page number within segment */
+){
+  if( iPgno!=1 ){
+    assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE );
+    if( p->pDeleteFromIdx==0 ){
+      fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf(
+          "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)",
+          p->pConfig->zDb, p->pConfig->zName
+      ));
+    }
+    if( p->rc==SQLITE_OK ){
+      sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid);
+      sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno);
+      sqlite3_step(p->pDeleteFromIdx);
+      p->rc = sqlite3_reset(p->pDeleteFromIdx);
+    }
+  }
+}
+
+/*
+** This is called when a secure-delete operation removes a position-list
+** that overflows onto segment page iPgno of segment pSeg. This function
+** rewrites node iPgno, and possibly one or more of its right-hand peers,
+** to remove this portion of the position list.
+**
+** Output variable (*pbLastInDoclist) is set to true if the position-list
+** removed is followed by a new term or the end-of-segment, or false if
+** it is followed by another rowid/position list.
+*/
+static void fts5SecureDeleteOverflow(
+  Fts5Index *p,
+  Fts5StructureSegment *pSeg,
+  int iPgno,
+  int *pbLastInDoclist
+){
+  const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE);
+  int pgno;
+  Fts5Data *pLeaf = 0;
+  assert( iPgno!=1 );
+
+  *pbLastInDoclist = 1;
+  for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){
+    i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno);
+    int iNext = 0;
+    u8 *aPg = 0;
+
+    pLeaf = fts5DataRead(p, iRowid);
+    if( pLeaf==0 ) break;
+    aPg = pLeaf->p;
+
+    iNext = fts5GetU16(&aPg[0]);
+    if( iNext!=0 ){
+      *pbLastInDoclist = 0;
+    }
+    if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){
+      fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext);
+    }
+
+    if( iNext==0 ){
+      /* The page contains no terms or rowids. Replace it with an empty
+      ** page and move on to the right-hand peer.  */
+      const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; 
+      assert_nc( bDetailNone==0 || pLeaf->nn==4 );
+      if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
+      fts5DataRelease(pLeaf);
+      pLeaf = 0;
+    }else if( bDetailNone ){
+      break;
+    }else if( iNext>=pLeaf->szLeaf || iNext<4 ){
+      p->rc = FTS5_CORRUPT;
+      break;
+    }else{
+      int nShift = iNext - 4;
+      int nPg;
+
+      int nIdx = 0;
+      u8 *aIdx = 0;
+
+      /* Unless the current page footer is 0 bytes in size (in which case
+      ** the new page footer will be as well), allocate and populate a 
+      ** buffer containing the new page footer. Set stack variables aIdx 
+      ** and nIdx accordingly.  */
+      if( pLeaf->nn>pLeaf->szLeaf ){
+        int iFirst = 0;
+        int i1 = pLeaf->szLeaf;
+        int i2 = 0;
+
+        aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2);
+        if( aIdx==0 ) break;
+        i1 += fts5GetVarint32(&aPg[i1], iFirst);
+        i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift);
+        if( i1<pLeaf->nn ){
+          memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1);
+          i2 += (pLeaf->nn-i1);
+        }
+        nIdx = i2;
+      }
+
+      /* Modify the contents of buffer aPg[]. Set nPg to the new size 
+      ** in bytes. The new page is always smaller than the old.  */
+      nPg = pLeaf->szLeaf - nShift;
+      memmove(&aPg[4], &aPg[4+nShift], nPg-4);
+      fts5PutU16(&aPg[2], nPg);
+      if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4);
+      if( nIdx>0 ){
+        memcpy(&aPg[nPg], aIdx, nIdx);
+        nPg += nIdx;
+      }
+      sqlite3_free(aIdx);
+
+      /* Write the new page to disk and exit the loop */
+      assert( nPg>4 || fts5GetU16(aPg)==0 );
+      fts5DataWrite(p, iRowid, aPg, nPg);
+      break;
+    }
+  }
+  fts5DataRelease(pLeaf);
+}
+
+/*
+** Completely remove the entry that pSeg currently points to from 
+** the database.
+*/
+static void fts5DoSecureDelete(
+  Fts5Index *p,
+  Fts5SegIter *pSeg
+){
+  const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE);
+  int iSegid = pSeg->pSeg->iSegid;
+  u8 *aPg = pSeg->pLeaf->p;
+  int nPg = pSeg->pLeaf->nn;
+  int iPgIdx = pSeg->pLeaf->szLeaf;
+
+  u64 iDelta = 0;
+  u64 iNextDelta = 0;
+  int iNextOff = 0;
+  int iOff = 0;
+  int nIdx = 0;
+  u8 *aIdx = 0;
+  int bLastInDoclist = 0;
+  int iIdx = 0;
+  int iStart = 0;
+  int iKeyOff = 0;
+  int iPrevKeyOff = 0;
+  int iDelKeyOff = 0;       /* Offset of deleted key, if any */
+
+  nIdx = nPg-iPgIdx;
+  aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16);
+  if( p->rc ) return;
+  memcpy(aIdx, &aPg[iPgIdx], nIdx);
+
+  /* At this point segment iterator pSeg points to the entry
+  ** this function should remove from the b-tree segment. 
+  **
+  ** In detail=full or detail=column mode, pSeg->iLeafOffset is the 
+  ** offset of the first byte in the position-list for the entry to 
+  ** remove. Immediately before this comes two varints that will also
+  ** need to be removed:
+  **
+  **     + the rowid or delta rowid value for the entry, and
+  **     + the size of the position list in bytes.
+  **
+  ** Or, in detail=none mode, there is a single varint prior to 
+  ** pSeg->iLeafOffset - the rowid or delta rowid value.
+  **
+  ** This block sets the following variables:
+  **
+  **   iStart:
+  **   iDelta:
+  */
+  {
+    int iSOP;
+    if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
+      iStart = pSeg->iTermLeafOffset;
+    }else{
+      iStart = fts5GetU16(&aPg[0]);
+    }
+
+    iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+    assert_nc( iSOP<=pSeg->iLeafOffset );
+
+    if( bDetailNone ){
+      while( iSOP<pSeg->iLeafOffset ){
+        if( aPg[iSOP]==0x00 ) iSOP++;
+        if( aPg[iSOP]==0x00 ) iSOP++;
+        iStart = iSOP;
+        iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+      }
+
+      iNextOff = iSOP;
+      if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
+      if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
+
+    }else{
+      int nPos = 0;
+      iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
+      while( iSOP<pSeg->iLeafOffset ){
+        iStart = iSOP + (nPos/2);
+        iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+        iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
+      }
+      assert_nc( iSOP==pSeg->iLeafOffset );
+      iNextOff = pSeg->iLeafOffset + pSeg->nPos;
+    }
+  }
+
+  iOff = iStart;
+  if( iNextOff>=iPgIdx ){
+    int pgno = pSeg->iLeafPgno+1;
+    fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
+    iNextOff = iPgIdx;
+  }else{
+    /* Set bLastInDoclist to true if the entry being removed is the last
+    ** in its doclist.  */
+    for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
+      u32 iVal = 0;
+      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+      iKeyOff += iVal;
+      if( iKeyOff==iNextOff ){
+        bLastInDoclist = 1;
+      }
+    }
+  }
+
+  if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){
+    fts5PutU16(&aPg[0], 0);
+  }
+
+  if( bLastInDoclist==0 ){
+    if( iNextOff!=iPgIdx ){
+      iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
+      iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
+    }
+  }else if( 
+      iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno 
+  ){
+    /* The entry being removed was the only position list in its
+    ** doclist. Therefore the term needs to be removed as well. */
+    int iKey = 0;
+    for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
+      u32 iVal = 0;
+      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+      if( (iKeyOff+iVal)>(u32)iStart ) break;
+      iKeyOff += iVal;
+    }
+
+    iDelKeyOff = iOff = iKeyOff;
+    if( iNextOff!=iPgIdx ){
+      int nPrefix = 0;
+      int nSuffix = 0;
+      int nPrefix2 = 0;
+      int nSuffix2 = 0;
+
+      iDelKeyOff = iNextOff;
+      iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2);
+      iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2);
+
+      if( iKey!=1 ){
+        iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix);
+      }
+      iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix);
+
+      nPrefix = MIN(nPrefix, nPrefix2);
+      nSuffix = (nPrefix2 + nSuffix2) - nPrefix;
+
+      if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){
+        p->rc = FTS5_CORRUPT;
+      }else{
+        if( iKey!=1 ){
+          iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix);
+        }
+        iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix);
+        if( nPrefix2>nPrefix ){
+          memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix);
+          iOff += (nPrefix2-nPrefix);
+        }
+        memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2);
+        iOff += nSuffix2;
+        iNextOff += nSuffix2;
+      }
+    }
+  }else if( iStart==4 ){
+      int iPgno;
+
+      assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
+      /* The entry being removed may be the only position list in
+      ** its doclist. */
+      for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
+        Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
+        int bEmpty = (pPg && pPg->nn==4);
+        fts5DataRelease(pPg);
+        if( bEmpty==0 ) break;
+      }
+
+      if( iPgno==pSeg->iTermLeafPgno ){
+        i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
+        Fts5Data *pTerm = fts5DataRead(p, iId);
+        if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
+          u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
+          int nTermIdx = pTerm->nn - pTerm->szLeaf;
+          int iTermIdx = 0;
+          int iTermOff = 0;
+
+          while( 1 ){
+            u32 iVal = 0;
+            int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
+            iTermOff += iVal;
+            if( (iTermIdx+nByte)>=nTermIdx ) break;
+            iTermIdx += nByte;
+          }
+          nTermIdx = iTermIdx;
+
+          memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
+          fts5PutU16(&pTerm->p[2], iTermOff);
+
+          fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
+          if( nTermIdx==0 ){
+            fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
+          }
+        }
+        fts5DataRelease(pTerm);
+      }
+    }
+
+    if( p->rc==SQLITE_OK ){
+      const int nMove = nPg - iNextOff;
+      int nShift = 0;
+
+      memmove(&aPg[iOff], &aPg[iNextOff], nMove);
+      iPgIdx -= (iNextOff - iOff);
+      nPg = iPgIdx;
+      fts5PutU16(&aPg[2], iPgIdx);
+
+      nShift = iNextOff - iOff;
+      for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
+        u32 iVal = 0;
+        iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+        iKeyOff += iVal;
+        if( iKeyOff!=iDelKeyOff ){
+          if( iKeyOff>iOff ){
+            iKeyOff -= nShift;
+            nShift = 0;
+          }
+          nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
+          iPrevKeyOff = iKeyOff;
+        }
+      }
+
+      if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
+        fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
+      }
+
+      assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
+      fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg);
+    }
+    sqlite3_free(aIdx);
+}
+
+/*
+** This is called as part of flushing a delete to disk in 'secure-delete'
+** mode. It edits the segments within the database described by argument
+** pStruct to remove the entries for term zTerm, rowid iRowid.
+*/
+static void fts5FlushSecureDelete(
+  Fts5Index *p,
+  Fts5Structure *pStruct,
+  const char *zTerm,
+  i64 iRowid
+){
+  const int f = FTS5INDEX_QUERY_SKIPHASH;
+  int nTerm = (int)strlen(zTerm);
+  Fts5Iter *pIter = 0;            /* Used to find term instance */
+
+  fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter);
+  if( fts5MultiIterEof(p, pIter)==0 ){
+    i64 iThis = fts5MultiIterRowid(pIter);
+    if( iThis<iRowid ){
+      fts5MultiIterNextFrom(p, pIter, iRowid);
+    }
+
+    if( p->rc==SQLITE_OK 
+     && fts5MultiIterEof(p, pIter)==0 
+     && iRowid==fts5MultiIterRowid(pIter)
+    ){
+      Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
+      fts5DoSecureDelete(p, pSeg);
+    }
+  }
+
+  fts5MultiIterFree(pIter);
+}
+
+
 /*
 ** Flush the contents of in-memory hash table iHash to a new level-0 
 ** segment on disk. Also update the corresponding structure record.
@@ -4587,6 +4983,7 @@ static void fts5FlushOneHash(Fts5Index *p){
   if( iSegid ){
     const int pgsz = p->pConfig->pgsz;
     int eDetail = p->pConfig->eDetail;
+    int bSecureDelete = p->pConfig->bSecureDelete;
     Fts5StructureSegment *pSeg;   /* New segment within pStruct */
     Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
     Fts5Buffer *pPgidx;           /* Buffer in which to assemble pgidx */
@@ -4609,40 +5006,77 @@ static void fts5FlushOneHash(Fts5Index *p){
     }
     while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){
       const char *zTerm;          /* Buffer containing term */
+      int nTerm;                  /* Size of zTerm in bytes */
       const u8 *pDoclist;         /* Pointer to doclist for this term */
       int nDoclist;               /* Size of doclist in bytes */
 
-      /* Write the term for this entry to disk. */
+      /* Get the term and doclist for this entry. */
       sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
-      fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm);
-      if( p->rc!=SQLITE_OK ) break;
+      nTerm = (int)strlen(zTerm);
+      if( bSecureDelete==0 ){
+        fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
+        if( p->rc!=SQLITE_OK ) break;
+        assert( writer.bFirstRowidInPage==0 );
+      }
 
-      assert( writer.bFirstRowidInPage==0 );
-      if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
+      if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
         /* The entire doclist will fit on the current leaf. */
         fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
       }else{
+        int bTermWritten = !bSecureDelete;
         i64 iRowid = 0;
-        u64 iDelta = 0;
+        i64 iPrev = 0;
         int iOff = 0;
 
         /* The entire doclist will not fit on this leaf. The following 
         ** loop iterates through the poslists that make up the current 
         ** doclist.  */
         while( p->rc==SQLITE_OK && iOff<nDoclist ){
+          u64 iDelta = 0;
           iOff += fts5GetVarint(&pDoclist[iOff], &iDelta);
           iRowid += iDelta;
+
+          /* If in secure delete mode, and if this entry in the poslist is
+          ** in fact a delete, then edit the existing segments directly
+          ** using fts5FlushSecureDelete().  */
+          if( bSecureDelete ){
+            if( eDetail==FTS5_DETAIL_NONE ){
+              if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+                iOff++;
+                if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                  iOff++;
+                  nDoclist = 0;
+                }else{
+                  continue;
+                }
+              }
+            }else if( (pDoclist[iOff] & 0x01) ){
+              fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+              if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){
+                iOff++;
+                continue;
+              }
+            }
+          }
+
+          if( p->rc==SQLITE_OK && bTermWritten==0 ){
+            fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
+            bTermWritten = 1;
+            assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 );
+          }
           
           if( writer.bFirstRowidInPage ){
             fts5PutU16(&pBuf->p[0], (u16)pBuf->n);   /* first rowid on page */
             pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
             writer.bFirstRowidInPage = 0;
             fts5WriteDlidxAppend(p, &writer, iRowid);
-            if( p->rc!=SQLITE_OK ) break;
           }else{
-            pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta);
+            pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev);
           }
+          if( p->rc!=SQLITE_OK ) break;
           assert( pBuf->n<=pBuf->nSpace );
+          iPrev = iRowid;
 
           if( eDetail==FTS5_DETAIL_NONE ){
             if( iOff<nDoclist && pDoclist[iOff]==0 ){
@@ -4701,20 +5135,23 @@ static void fts5FlushOneHash(Fts5Index *p){
     sqlite3Fts5HashClear(pHash);
     fts5WriteFinish(p, &writer, &pgnoLast);
 
-    /* Update the Fts5Structure. It is written back to the database by the
-    ** fts5StructureRelease() call below.  */
-    if( pStruct->nLevel==0 ){
-      fts5StructureAddLevel(&p->rc, &pStruct);
+    assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 );
+    if( pgnoLast>0 ){
+      /* Update the Fts5Structure. It is written back to the database by the
+      ** fts5StructureRelease() call below.  */
+      if( pStruct->nLevel==0 ){
+        fts5StructureAddLevel(&p->rc, &pStruct);
+      }
+      fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
+      if( p->rc==SQLITE_OK ){
+        pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
+        pSeg->iSegid = iSegid;
+        pSeg->pgnoFirst = 1;
+        pSeg->pgnoLast = pgnoLast;
+        pStruct->nSegment++;
+      }
+      fts5StructurePromote(p, 0, pStruct);
     }
-    fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
-    if( p->rc==SQLITE_OK ){
-      pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
-      pSeg->iSegid = iSegid;
-      pSeg->pgnoFirst = 1;
-      pSeg->pgnoLast = pgnoLast;
-      pStruct->nSegment++;
-    }
-    fts5StructurePromote(p, 0, pStruct);
   }
 
   fts5IndexAutomerge(p, &pStruct, pgnoLast);
@@ -5455,6 +5892,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p){
     sqlite3_finalize(p->pIdxDeleter);
     sqlite3_finalize(p->pIdxSelect);
     sqlite3_finalize(p->pDataVersion);
+    sqlite3_finalize(p->pDeleteFromIdx);
     sqlite3Fts5HashFree(p->pHash);
     sqlite3_free(p->zDataTbl);
     sqlite3_free(p);
@@ -6085,6 +6523,7 @@ static void fts5IndexIntegrityCheckSegment(
   Fts5StructureSegment *pSeg      /* Segment to check internal consistency */
 ){
   Fts5Config *pConfig = p->pConfig;
+  int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE);
   sqlite3_stmt *pStmt = 0;
   int rc2;
   int iIdxPrevLeaf = pSeg->pgnoFirst-1;
@@ -6120,7 +6559,19 @@ static void fts5IndexIntegrityCheckSegment(
     ** is also a rowid pointer within the leaf page header, it points to a
     ** location before the term.  */
     if( pLeaf->nn<=pLeaf->szLeaf ){
-      p->rc = FTS5_CORRUPT;
+
+      if( nIdxTerm==0 
+       && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE
+       && pLeaf->nn==pLeaf->szLeaf
+       && pLeaf->nn==4
+      ){
+        /* special case - the very first page in a segment keeps its %_idx
+        ** entry even if all the terms are removed from it by secure-delete 
+        ** operations. */
+      }else{
+        p->rc = FTS5_CORRUPT;
+      }
+
     }else{
       int iOff;                   /* Offset of first term on leaf */
       int iRowidOff;              /* Offset of first rowid on leaf */
@@ -6184,9 +6635,12 @@ static void fts5IndexIntegrityCheckSegment(
           ASSERT_SZLEAF_OK(pLeaf);
           if( iRowidOff>=pLeaf->szLeaf ){
             p->rc = FTS5_CORRUPT;
-          }else{
+          }else if( bSecureDelete==0 || iRowidOff>0 ){
+            i64 iDlRowid = fts5DlidxIterRowid(pDlidx);
             fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
-            if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT;
+            if( iRowid<iDlRowid || (bSecureDelete==0 && iRowid!=iDlRowid) ){
+              p->rc = FTS5_CORRUPT;
+            }
           }
           fts5DataRelease(pLeaf);
         }

ext/fts5/fts5_main.c

@@ -1623,6 +1623,8 @@ static int fts5UpdateMethod(
   Fts5Config *pConfig = pTab->p.pConfig;
   int eType0;                     /* value_type() of apVal[0] */
   int rc = SQLITE_OK;             /* Return code */
+  int bUpdateOrDelete = 0;
+  
 
   /* A transaction must be open when this is called. */
   assert( pTab->ts.eState==1 || pTab->ts.eState==2 );
@@ -1633,6 +1635,11 @@ static int fts5UpdateMethod(
        || sqlite3_value_type(apVal[0])==SQLITE_NULL 
   );
   assert( pTab->p.pConfig->pzErrmsg==0 );
+  if( pConfig->pgsz==0 ){
+    rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
+    if( rc!=SQLITE_OK ) return rc;
+  }
+
   pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
 
   /* Put any active cursors into REQUIRE_SEEK state. */
@@ -1685,6 +1692,7 @@ static int fts5UpdateMethod(
     else if( nArg==1 ){
       i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
       rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
+      bUpdateOrDelete = 1;
     }
 
     /* INSERT or UPDATE */
@@ -1700,6 +1708,7 @@ static int fts5UpdateMethod(
         if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){
           i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
           rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
+          bUpdateOrDelete = 1;
         }
         fts5StorageInsert(&rc, pTab, apVal, pRowid);
       }
@@ -1728,10 +1737,24 @@ static int fts5UpdateMethod(
           rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
           fts5StorageInsert(&rc, pTab, apVal, pRowid);
         }
+        bUpdateOrDelete = 1;
       }
     }
   }
 
+  if( rc==SQLITE_OK 
+   && bUpdateOrDelete 
+   && pConfig->bSecureDelete 
+   && pConfig->iVersion==FTS5_CURRENT_VERSION 
+  ){
+    rc = sqlite3Fts5StorageConfigValue(
+        pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE
+    );
+    if( rc==SQLITE_OK ){
+      pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE;
+    }
+  }
+
   pTab->p.pConfig->pzErrmsg = 0;
   return rc;
 }
@@ -2591,6 +2614,7 @@ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
   UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
   fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
   fts5TripCursors(pTab);
+  pTab->p.pConfig->pgsz = 0;
   return sqlite3Fts5StorageRollback(pTab->pStorage);
 }
 

ext/fts5/test/fts5_common.tcl

@@ -594,6 +594,10 @@ proc nearset_rc {aCol args} {
   list
 }
 
+proc dump {tname} {
+  execsql_pp "SELECT * FROM ${tname}_idx"
+  execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data"
+}
 
 #-------------------------------------------------------------------------
 # Code for a simple Tcl tokenizer that supports synonyms at query time.

ext/fts5/test/fts5corrupt7.test

@@ -0,0 +1,99 @@
+# 2023 April 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5corrupt7
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+sqlite3_fts5_may_be_corrupt 1
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+}
+
+set doc [string repeat "a b " 30]
+
+do_execsql_test 1.1 {
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(123, $doc);
+    INSERT INTO t1(rowid, x) VALUES(124, $doc);
+  COMMIT;
+}
+
+execsql_pp {
+  SELECT id, fts5_decode(id, block), quote(block) FROM t1_data
+}
+
+set rows [db eval { SELECT rowid FROM t1_data }]
+db_save_and_close
+
+foreach r $rows {
+  db_restore_and_reopen
+
+  proc edit_block {b} {
+    binary scan $b c* in
+    set out [lreplace $in 0 1 255 255]
+    binary format c* $out
+  }
+  db func edit_block edit_block
+
+  do_execsql_test 1.2.$r.1 {
+    UPDATE t1_data SET block = edit_block(block) WHERE rowid=$r;
+  }
+
+  do_execsql_test 1.2.$r.2 {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+
+  do_test 1.2.$r.3 {
+    catchsql { DELETE FROM t1 WHERE rowid=123; }
+    catchsql { DELETE FROM t1 WHERE rowid=124; }
+    set {} {}
+  } {}
+
+  db close
+}
+
+foreach r $rows {
+set r 137438953475
+  db_restore_and_reopen
+
+  proc edit_block {b} {
+    binary scan $b c* in
+    set out [lreplace $in end end 127]
+    binary format c* $out
+  }
+  db func edit_block edit_block
+
+  do_execsql_test 1.2.$r.1 {
+    UPDATE t1_data SET block = edit_block(block) WHERE rowid=$r;
+  }
+
+  do_execsql_test 1.2.$r.2 {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+
+  do_test 1.2.$r.3 {
+    catchsql { DELETE FROM t1 WHERE rowid=124; }
+    catchsql { DELETE FROM t1 WHERE rowid=123; }
+    set {} {}
+  } {}
+
+  db close
+}
+
+finish_test

ext/fts5/test/fts5misc.test

@@ -443,5 +443,33 @@ do_execsql_test -db db2 16.6 {
   SELECT * FROM x1
 } {abc def}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 17.1 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="unicode61 separators 'X'");
+}
+do_execsql_test 17.2 {
+  SELECT 0 FROM ft WHERE ft MATCH 'X' AND ft MATCH 'X'
+}
+do_execsql_test 17.3 {
+  SELECT 0 FROM ft('X')
+}
+
+do_execsql_test 17.4 {
+  CREATE VIRTUAL TABLE t0 USING fts5(c0, t="trigram");
+  INSERT INTO t0 VALUES('assertionfaultproblem');
+}
+do_execsql_test 17.5 {
+  SELECT 0 FROM t0(0) WHERE c0 GLOB 0;
+} {}
+
+do_execsql_test 17.5 {
+  SELECT c0 FROM t0 WHERE c0 GLOB '*f*';
+} {assertionfaultproblem}
+do_execsql_test 17.5 {
+  SELECT c0 FROM t0 WHERE c0 GLOB '*faul*';
+} {assertionfaultproblem}
+
+
 finish_test
 

ext/fts5/test/fts5secure.test

@@ -0,0 +1,278 @@
+# 2023 Feb 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure
+
+proc dump {tname} {
+  execsql_pp "SELECT * FROM ${tname}_idx"
+  execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data"
+}
+
+
+do_execsql_test 0.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(ab);
+  CREATE VIRTUAL TABLE v1 USING fts5vocab('t1', 'instance');
+  INSERT INTO t1(rowid, ab) VALUES
+      (0,'abc'), (1,'abc'), (2,'abc'), (3,'abc'), (4,'def');
+}
+
+do_execsql_test 0.1 {
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 0.2 {
+  DELETE FROM t1 WHERE rowid=2;
+}
+
+do_execsql_test 0.3 {
+  SELECT count(*) FROM t1_data
+} 3
+
+do_execsql_test 0.4 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} 
+
+do_execsql_test 0.5 {
+  DELETE FROM t1 WHERE rowid=3;
+}
+
+do_execsql_test 0.6 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} 
+
+do_execsql_test 0.7 {
+  DELETE FROM t1 WHERE rowid=0;
+}
+
+do_execsql_test 0.8 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} 
+
+#----------------------------------
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(ab);
+  INSERT INTO t2(rowid, ab) VALUES (5, 'key'), (6, 'value');
+  INSERT INTO t2(t2, rank) VALUES('secure-delete', 1);
+}
+
+#execsql_pp { SELECT id, quote(block) FROM t1_data }
+#execsql_pp { SELECT segid, quote(term), pgno FROM t1_idx }
+
+do_execsql_test 1.1 {
+  DELETE FROM t2 WHERE rowid = 5;
+}
+
+do_execsql_test 1.2 {
+  INSERT INTO t2(t2) VALUES('integrity-check');
+} 
+
+do_execsql_test 1.3 {
+  DELETE FROM t2 WHERE rowid = 6;
+}
+
+do_execsql_test 1.4 {
+  INSERT INTO t2(t2) VALUES('integrity-check');
+} 
+
+do_execsql_test 1.5 {
+  SELECT * FROM t2('value');
+  SELECT * FROM t2('v*');
+} 
+
+do_execsql_test 1.6 {
+  SELECT * FROM t2('value') ORDER BY rowid DESC;
+  SELECT * FROM t2('v*') ORDER BY rowid DESC;
+} 
+execsql_pp {
+  SELECT id, quote(block) FROM t2_data;
+}
+
+#----------------------------------
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(ab);
+  CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', 'instance');
+  INSERT INTO ft(rowid, ab) VALUES
+    (1, 'one'), 
+    (2, 'two'), 
+    (3, 'three'), 
+    (4, 'four'), 
+    (5, 'one one'), 
+    (6, 'one two'), 
+    (7, 'one three'), 
+    (8, 'one four'), 
+    (9,  'two one'),
+    (10, 'two two'),
+    (11, 'two three'),
+    (12, 'two four'),
+    (13, 'three one'), 
+    (14, 'three two'), 
+    (15, 'three three'), 
+    (16, 'three four');
+}
+
+do_execsql_test 2.1 {
+  SELECT count(*) FROM ft_data;
+} {3}
+
+do_execsql_test 2.2 {
+  INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 2.3 {
+  DELETE FROM ft WHERE rowid=9;
+}
+
+do_execsql_test 2.4 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.5 {
+  DELETE FROM ft WHERE ab LIKE '%two%'
+}
+
+do_execsql_test 2.6 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.7 {
+  SELECT count(*) FROM ft_data;
+} {3}
+
+#----------------------------------
+reset_db
+
+set ::vocab {
+  one two three four five six seven eight nine ten
+  eleven twelve thirteen fourteen fifteen sixteen 
+  seventeen eighteen nineteen twenty
+}
+proc rnddoc {} {
+  set nVocab [llength $::vocab]
+  set ret [list]
+  for {set ii 0} {$ii < 8} {incr ii} {
+    lappend ret [lindex $::vocab [expr int(abs(rand()) * $nVocab)]]
+  }
+  set ret
+}
+
+proc contains {list val} {
+  expr {[lsearch $list $val]>=0}
+}
+
+foreach {tn pgsz} {
+  2 64
+  1 1000
+} {
+  reset_db
+  db function rnddoc rnddoc
+  db function contains contains
+  
+  expr srand(1)
+
+  do_execsql_test 3.$tn.0 {
+    CREATE VIRTUAL TABLE t1 USING fts5(x);
+    INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
+    WITH s(i) AS (
+      VALUES(1) UNION SELECT i+1 FROM s WHERE i<20
+    )
+    INSERT INTO t1 SELECT rnddoc() FROM s;
+  }
+
+  do_execsql_test 3.$tn.1 {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+  
+  foreach {rowid} {
+    6 16 3 4 9 14 13 7 20 15 19 10 11 2 5 18 17 1 12 8
+  } {
+
+    do_execsql_test 3.$tn.2.$rowid {
+      DELETE FROM t1 WHERE rowid=$rowid;
+    }
+    do_execsql_test 3.$tn.2.$rowid.ic {
+      INSERT INTO t1(t1) VALUES('integrity-check');
+    }
+
+    foreach v $::vocab {
+      do_execsql_test 3.$tn.2.$rowid.q.$v {
+        SELECT rowid FROM t1($v)
+      } [db eval {SELECT rowid FROM t1 WHERE contains(x, $v)}]
+
+      do_execsql_test 3.$tn.2.$rowid.q.$v.DESC {
+        SELECT rowid FROM t1($v) ORDER BY 1 DESC
+      } [db eval {SELECT rowid FROM t1 WHERE contains(x, $v) ORDER BY 1 DESC}]
+    }
+  }
+}
+
+do_execsql_test 3.3 {
+  INSERT INTO t1(x) VALUES('optimize');
+  INSERT INTO t1(t1) VALUES('optimize');
+  SELECT count(*) FROM t1_data;
+} {3}
+
+#----------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+
+set L1 [string repeat abcdefghij 10]
+set L2 [string repeat 1234567890 10]
+
+do_execsql_test 4.1 {
+  INSERT INTO t1 VALUES('aa' || $L1 || ' ' || $L2);
+}
+do_execsql_test 4.2 {
+  DELETE FROM t1 WHERE rowid=1
+}
+do_execsql_test 4.3 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+#----------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+
+set doc "aa [string repeat {abc } 60]"
+
+do_execsql_test 5.1 {
+  BEGIN;
+    INSERT INTO t1 VALUES($doc);
+    INSERT INTO t1 VALUES('aa abc');
+  COMMIT;
+}
+
+do_execsql_test 5.2 {
+  DELETE FROM t1 WHERE rowid = 1;
+}
+
+do_execsql_test 5.3 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2
+do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2
+
+
+finish_test
+

ext/fts5/test/fts5secure2.test

@@ -0,0 +1,87 @@
+# 2023 Feb 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure2
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(col);
+  INSERT INTO ft VALUES('data for the table');
+  INSERT INTO ft VALUES('more of the same');
+  INSERT INTO ft VALUES('and extra data');
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM ft_config
+} {version 4}
+
+do_execsql_test 1.2 {
+  INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
+  SELECT * FROM ft_config;
+} {secure-delete 1 version 4}
+
+do_execsql_test 1.3 {
+  INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
+  SELECT * FROM ft_config;
+} {secure-delete 1 version 4}
+
+do_execsql_test 1.4 {
+  DELETE FROM ft WHERE rowid=2;
+  SELECT * FROM ft_config;
+} {secure-delete 1 version 5}
+
+do_execsql_test 1.5 {
+  SELECT rowid, col FROM ft('data');
+} {1 {data for the table} 3 {and extra data}}
+
+db close
+sqlite3 db test.db
+
+do_execsql_test 1.6 {
+  SELECT rowid, col FROM ft('data');
+} {1 {data for the table} 3 {and extra data}}
+
+#------------------------------------------------------------------------
+
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(col);
+  INSERT INTO ft VALUES('one zero one one zero');
+  INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 2.1 {
+  SELECT count(*) FROM ft_data WHERE block=X'00000004';
+} {0}
+
+do_execsql_test 2.2 {
+  UPDATE ft SET col = 'zero one zero zero one' WHERE rowid=1;
+}
+
+do_execsql_test 2.3 {
+  SELECT count(*) FROM ft_data WHERE block=X'00000004';
+} {1}
+
+do_execsql_test 2.4 {
+  INSERT INTO ft VALUES('one zero zero one');
+  DELETE FROM ft WHERE rowid=1;
+}
+
+do_execsql_test 2.5 {
+  SELECT count(*) FROM ft_data WHERE block=X'00000004';
+} {2}
+
+
+finish_test
+
+

ext/fts5/test/fts5secure3.test

@@ -0,0 +1,166 @@
+# 2023 Feb 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# TESTRUNNER: slow
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure3
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(col);
+  INSERT INTO ft VALUES('data for the table');
+  INSERT INTO ft VALUES('more of the same');
+  INSERT INTO ft VALUES('and extra data');
+
+  INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+    INSERT INTO ft(rowid, col) VALUES(0, 'the next data');
+    DELETE FROM ft WHERE rowid=1;
+    DELETE FROM ft WHERE rowid=2;
+    INSERT INTO ft(rowid, col) VALUES(6, 'with some more of the same data');
+  COMMIT;
+}
+
+do_execsql_test 1.2 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  BEGIN;
+    INSERT INTO t1 VALUES('the start');
+}
+do_test 2.1 {
+  for {set i 0} {$i < 1000} {incr i} {
+    execsql { INSERT INTO t1 VALUES('the ' || hex(randomblob(3))) }
+  }
+  execsql  {
+      INSERT INTO t1 VALUES('the end');
+    COMMIT;
+  }
+} {}
+
+do_execsql_test 2.2 {
+  DELETE FROM t1 WHERE rowid BETWEEN 2 AND 1000;
+}
+
+do_execsql_test 2.3 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_execsql_test 2.6 {
+  INSERT INTO t1(rowid, x) VALUES(500, 'middle');
+  INSERT INTO t1(rowid, x) VALUES(501, 'value');
+  SELECT * FROM t1('the middle');
+}
+
+do_execsql_test 2.7 {
+  INSERT INTO t1(t1) VALUES('optimize');
+}
+
+do_execsql_test 2.8 {
+  SELECT count(*) FROM t1_data
+} 4
+
+#execsql_pp { SELECT id, quote(block), fts5_decode(id, block) FROM t1_data; }
+
+#-------------------------------------------------------------------------
+# Tests with large/small rowid values.
+#
+
+reset_db
+
+expr srand(0)
+
+set vocab {
+  Popper Poppins Popsicle Porfirio Porrima Porsche
+  Porter Portia Portland Portsmouth Portugal Portuguese
+  Poseidon Post PostgreSQL Potemkin Potomac Potsdam
+  Pottawatomie Potter Potts Pound Poussin Powell
+  PowerPC PowerPoint Powers Powhatan Poznan Prada
+  Prado Praetorian Prague Praia Prakrit Pratchett
+  Pratt Pravda Praxiteles Preakness Precambrian Preminger
+  Premyslid Prensa Prentice Pres Presbyterian Presbyterianism
+}
+proc newdoc {} {
+  for {set i 0} {$i<8} {incr i} {
+    lappend ret [lindex $::vocab [expr int(abs(rand()) * [llength $::vocab])]]
+  }
+  set ret
+}
+db func newdoc newdoc
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE fff USING fts5(y);
+  INSERT INTO fff(fff, rank) VALUES('pgsz', 64);
+
+  WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 )
+  INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s;
+
+  WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 )
+  INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s;
+
+  WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 )
+  INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s;
+
+  INSERT INTO fff(fff, rank) VALUES('secure-delete', 1);
+}
+
+proc lshuffle {in} {
+  set out [list]
+  while {[llength $in]>0} {
+    set idx [expr int(abs(rand()) * [llength $in])]
+    lappend out [lindex $in $idx]
+    set in [lreplace $in $idx $idx]
+  }
+  set out
+}
+
+#dump fff
+
+set iTest 1
+foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] {
+  #if {$iTest==1} { dump fff }
+  #if {$iTest==1} { breakpoint }
+  do_execsql_test 3.1.$iTest.$ii {
+    DELETE FROM fff WHERE rowid=$ii;
+  }
+  #if {$iTest==1} { dump fff }
+  if {($iTest % 20)==0} {
+    do_execsql_test 3.1.$iTest.$ii.ic {
+      INSERT INTO fff(fff) VALUES('integrity-check');
+    }
+  }
+  #if {$iTest==1} { break }
+  incr iTest
+}
+
+#execsql_pp { SELECT rowid FROM fff('post') ORDER BY rowid ASC }
+#breakpoint
+#execsql_pp { 
+#  SELECT rowid FROM fff('post') ORDER BY rowid DESC 
+#}
+#
+#dump fff
+
+
+finish_test
+

ext/fts5/test/fts5secure4.test

@@ -0,0 +1,170 @@
+# 2023 April 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+return_if_no_fts5
+set ::testprefix fts5secure4
+
+#-------------------------------------------------------------------------
+# Test using the 'delete' command to attempt to delete a token that 
+# is not present in the index in secure-delete mode.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, content=x1);
+
+  CREATE TABLE x1(rowid INTEGER PRIMARY KEY, a, b);
+  INSERT INTO x1 VALUES
+    (1, 'hello world', 'today xyz'),
+    (2, 'not the day', 'crunch crumble and chomp'),
+    (3, 'one', 'two');
+  INSERT INTO t1(t1) VALUES('rebuild');
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 1.2 {
+  INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 4, 'nosuchtoken', '');
+}
+
+do_execsql_test 1.3 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_execsql_test 1.4 {
+  INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 1, 'crunch', '');
+}
+
+do_execsql_test 1.5 {
+  INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 3, 'crunch', '');
+}
+
+do_execsql_test 1.6 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_execsql_test 1.7 {
+CREATE VIRTUAL TABLE y1 USING fts5(xx, prefix='1,2');
+INSERT INTO y1(y1, rank) VALUES('pgsz', 64);
+INSERT INTO y1(y1, rank) VALUES('secure-delete', 1);
+}
+do_execsql_test 1.8 {
+  BEGIN;
+  INSERT INTO y1(rowid, xx) VALUES(1, 'abc def');
+  INSERT INTO y1(rowid, xx) VALUES(2, 'reallyreallylongtoken');
+  COMMIT;
+}
+do_execsql_test 1.9 {
+  DELETE FROM y1 WHERE rowid=1;
+  INSERT INTO y1(y1) VALUES('integrity-check');
+}
+
+do_execsql_test 1.10 {
+  CREATE VIRTUAL TABLE w1 USING fts5(ww, content="");
+  INSERT INTO w1(rowid, ww) VALUES(123, '');
+}
+do_catchsql_test 1.11 {
+  INSERT INTO w1(w1, rowid, ww) VALUES('delete', 123, 'xyz');
+} {1 {database disk image is malformed}}
+do_catchsql_test 1.12 {
+  DROP TABLE w1;
+  CREATE VIRTUAL TABLE w1 USING fts5(ww, content="");
+  INSERT INTO w1(rowid, ww) VALUES(123, '');
+  DELETE FROM w1_data WHERE id>10;
+  INSERT INTO w1(w1, rowid, ww) VALUES('delete', 123, 'xyz');
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+# Test using secure-delete with detail=none or detail=col.
+#
+foreach {tn d} {1 full 2 none 3 column} {
+  reset_db
+  do_execsql_test 2.$tn.1 "
+    CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='10,20');
+    INSERT INTO x1(x1, rank) VALUES('pgsz', 64);
+    INSERT INTO x1(x1, rank) VALUES('secure-delete', 1);
+  "
+
+  do_execsql_test 2.$tn.2 {
+    BEGIN;
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
+    COMMIT;
+    INSERT INTO x1(x1) VALUES('integrity-check');
+  }
+
+  do_execsql_test 2.$tn.3 {
+    DELETE FROM x1 WHERE rowid IN (2, 4, 6);
+    INSERT INTO x1(x1) VALUES('integrity-check');
+  }
+
+  do_execsql_test 2.$tn.4 {
+    DELETE FROM x1 WHERE rowid IN (1, 3, 5);
+    INSERT INTO x1(x1) VALUES('integrity-check');
+  }
+
+  do_execsql_test 2.$tn.5 {
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+    )
+    INSERT INTO x1 
+      SELECT 'seems to be', 'used brew to', 'everything is working' FROM s
+        UNION ALL
+      SELECT 'used brew to', 'everything is working', 'seems to be' FROM s
+        UNION ALL
+      SELECT 'everything is working', 'seems to be', 'used brew to' FROM s
+        UNION ALL
+      SELECT 'abc', 'zzz', 'a b c d'
+        UNION ALL
+      SELECT 'z', 'z', 'z' FROM s
+  }
+
+  do_test 2.$tn.6 {
+    for {set i 300} {$i > 200} {incr i -1} {
+      execsql {
+        DELETE FROM x1 WHERE rowid=$i;
+        INSERT INTO x1(x1) VALUES('integrity-check');
+      }
+    }
+  } {}
+
+  do_test 2.$tn.7 {
+    for {set i 1} {$i < 100} {incr i} {
+      execsql {
+        DELETE FROM x1 WHERE rowid=$i;
+        INSERT INTO x1(x1) VALUES('integrity-check');
+      }
+    }
+  } {}
+
+  do_test 2.$tn.8 {
+    foreach i [db eval {SELECT rowid FROM x1}] {
+      execsql {
+        DELETE FROM x1 WHERE rowid=$i;
+        INSERT INTO x1(x1) VALUES('integrity-check');
+      }
+    }
+  } {}
+
+  do_execsql_test 2.$tn.9 {
+    SELECT * FROM x1
+  } {}
+}
+
+
+
+finish_test
+

ext/fts5/test/fts5secure5.test

@@ -0,0 +1,129 @@
+# 2023 April 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+return_if_no_fts5
+set ::testprefix fts5secure5
+return_if_no_fts5
+
+proc dump {} {
+  execsql_pp {
+    SELECT id, quote(block), fts5_decode_none(id, block) FROM ft1_data
+  }
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+    INSERT INTO ft1(rowid, a) VALUES(1, 'abcd');
+    INSERT INTO ft1(rowid, a) VALUES(2, 'abcd');
+    INSERT INTO ft1(rowid, a) VALUES(3, 'abcd');
+  COMMIT;
+}
+do_execsql_test 1.2 {
+  DELETE FROM ft1 WHERE rowid=1;
+}
+do_execsql_test 1.3 {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 1.4 {
+  DELETE FROM ft1 WHERE rowid=3;
+}
+do_execsql_test 1.5 {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 1.6 {
+  DELETE FROM ft1 WHERE rowid=3;
+}
+do_execsql_test 1.7 {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 2.1 {
+  BEGIN;
+    INSERT INTO ft1(rowid, a) VALUES(1, 'abcd one');
+    INSERT INTO ft1(rowid, a) VALUES(2, 'abcd two');
+    INSERT INTO ft1(rowid, a) VALUES(3, 'abcd two');
+    INSERT INTO ft1(rowid, a) VALUES(4, 'abcd two');
+    INSERT INTO ft1(rowid, a) VALUES(5, 'abcd three');
+  COMMIT;
+}
+
+do_execsql_test 2.2a {
+  DELETE FROM ft1 WHERE rowid=3;
+}
+do_execsql_test 2.2b {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 2.3a {
+  DELETE FROM ft1 WHERE rowid=2;
+}
+do_execsql_test 2.3b {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 2.4a {
+  DELETE FROM ft1 WHERE rowid=4;
+}
+do_execsql_test 2.4b {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none, prefix=1);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
+}
+do_execsql_test 3.1 {
+  BEGIN;
+    INSERT INTO ft1(a) VALUES('c');
+  COMMIT;
+}
+do_execsql_test 3.2 {
+  DELETE FROM ft1 WHERE rowid IN (1);
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
+
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500
+  )
+  INSERT INTO ft1 SELECT 'abcdefg' FROM s;
+}
+
+do_test 4.1 {
+  for {set i 500} {$i > 0} {incr i -1} {
+    execsql { DELETE FROM ft1 WHERE rowid=$i }
+    execsql { INSERT INTO ft1(ft1) VALUES('integrity-check') }
+  }
+} {}
+
+finish_test
+

ext/fts5/test/fts5secure6.test

@@ -0,0 +1,55 @@
+# 2023 Feb 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure6
+
+db progress 1 progress_handler
+set ::PHC 0
+proc progress_handler {args} {
+  incr ::PHC
+  if {($::PHC % 100000)==0}  breakpoint
+  return 0
+}
+
+proc setup {} {
+  db eval {
+    DROP TABLE IF EXISTS t1;
+    CREATE VIRTUAL TABLE t1 USING fts5(x);
+    WITH s(i) AS (
+      VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1000
+    )
+    INSERT INTO t1 SELECT 'a b c d e f g h i j k' FROM s;
+  }
+}
+
+foreach {tn sd} {
+  1 0 
+  2 1
+} {
+  setup
+  do_execsql_test 1.$tn.0 { 
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) 
+  }
+  set PHC 0
+  do_execsql_test 1.$tn.1 { DELETE FROM t1; }
+  set phc($tn) $PHC
+}
+
+do_test 1.3 {
+  expr $phc(1)*5 < $phc(2)
+} {1}
+
+
+finish_test
+

ext/fts5/test/fts5securefault.test

@@ -0,0 +1,225 @@
+# 2023 April 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5securefault
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+return_if_no_fts5
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(ab);
+  INSERT INTO t1(rowid, ab) VALUES
+      (0, 'abc'), (1, 'abc'), (2, 'abc'), (3, 'abc'), (4, 'def');
+}
+faultsim_save_and_close
+
+do_faultsim_test 1.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  execsql {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+} -body {
+  execsql { DELETE FROM t1 WHERE rowid=2 }
+} -test {
+  faultsim_test_result {0 {}}
+}
+do_faultsim_test 1.2 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  execsql {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+} -body {
+  execsql { DELETE FROM t1 WHERE rowid IN(0, 1, 2, 3, 4) }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+set big  [string repeat abcdefghij 5]
+set big2 [string repeat klmnopqrst 5]
+set doc "$big $big2"
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(ab);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<4
+  )
+  INSERT INTO t1(rowid, ab) SELECT i, $doc FROM s;
+}
+faultsim_save_and_close
+
+do_faultsim_test 2.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  execsql {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+} -body {
+  execsql { DELETE FROM t1 WHERE rowid = 3 }
+  execsql { DELETE FROM t1 WHERE rowid = 4 }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+set big  [string repeat abcdefghij 5]
+set big2 [string repeat klmnopqrst 5]
+set doc "$big $big2"
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(ab);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<25
+  )
+  INSERT INTO t1(rowid, ab) SELECT i, $doc FROM s;
+
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  DELETE FROM t1 WHERE rowid BETWEEN 3 AND 23;
+}
+faultsim_save_and_close
+
+do_faultsim_test 3.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  execsql {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+} -body {
+  execsql { DELETE FROM t1 WHERE rowid = 24 }
+  execsql { DELETE FROM t1 WHERE rowid = 25 }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+set doc [string repeat "tok " 400]
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(ab);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+  INSERT INTO t1(rowid, ab) VALUES(1, $doc), (2, $doc), (3, $doc);
+}
+faultsim_save_and_close
+
+do_faultsim_test 4.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  execsql {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+} -body {
+  execsql { DELETE FROM t1 WHERE rowid = 2 }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+
+set doc1 [string repeat "abc " 10]
+set doc2 [string repeat "def " 10]
+
+do_test 5.0 {
+  execsql {
+    CREATE VIRTUAL TABLE t1 USING fts5(ab);
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+    BEGIN;
+  }
+  for {set i 0} {$i < 50} {incr i} {
+    execsql {
+      INSERT INTO t1(rowid, ab) VALUES($i, 'abcdefg');
+    }
+  }
+  execsql {
+    INSERT INTO t1(rowid, ab) VALUES(105, 'def');
+    COMMIT;
+  }
+} {}
+faultsim_save_and_close
+
+do_faultsim_test 5.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  execsql {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+} -body {
+  execsql { DELETE FROM t1 WHERE rowid = 105 }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_test 6.0 {
+  execsql {
+    CREATE VIRTUAL TABLE t1 USING fts5(ab);
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+    BEGIN;
+      INSERT INTO t1(rowid, ab) VALUES(1, 'abcdefg');
+      INSERT INTO t1(rowid, ab) VALUES(2, 'abcdefg');
+      INSERT INTO t1(rowid, ab) VALUES(3, 'abcdefg');
+    COMMIT;
+  }
+} {}
+faultsim_save_and_close
+
+do_faultsim_test 6.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  execsql {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+} -body {
+  execsql { 
+    UPDATE t1 SET ab='abcdefg' WHERE rowid=2;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_test 7.0 {
+  execsql {
+    CREATE VIRTUAL TABLE t1 USING fts5(ab);
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+} {}
+faultsim_save_and_close
+
+do_faultsim_test 7.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  set big1 "[string repeat x 50] [string repeat y 50] [string repeat z 50]"
+  execsql {
+    BEGIN;
+      INSERT INTO t1 VALUES($big1);
+  }
+} -body {
+  execsql { COMMIT }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+
+finish_test

ext/fts5/test/fts5version.test

@@ -38,20 +38,20 @@ do_execsql_test 1.3 {
 
 sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 1.4 {
-  UPDATE t1_config set v=5 WHERE k='version';
+  UPDATE t1_config set v=6 WHERE k='version';
 }
 
 do_test 1.5 {
   db close
   sqlite3 db test.db
   catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
-} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}
+} {1 {invalid fts5 file format (found 6, expected 4 or 5) - run 'rebuild'}}
 
 do_test 1.6 {
   db close
   sqlite3 db test.db
   catchsql { INSERT INTO t1 VALUES('x y z') }
-} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}
+} {1 {invalid fts5 file format (found 6, expected 4 or 5) - run 'rebuild'}}
 
 do_test 1.7 {
   sqlite3_db_config db DEFENSIVE 0
@@ -59,7 +59,75 @@ do_test 1.7 {
   db close
   sqlite3 db test.db
   catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
-} {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}}
+} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}}
+
+do_test 1.8 {
+  sqlite3_db_config db DEFENSIVE 0
+  execsql { INSERT INTO t1_config VALUES('version', 4) }
+  execsql { INSERT INTO t1(t1, rank) VALUES('secure-delete', 1) }
+} {}
+
+do_execsql_test 1.10 {
+  SELECT * FROM t1_config
+} {secure-delete 1 version 4}
+
+do_execsql_test 1.11 {
+  INSERT INTO t1(rowid, one) VALUES(123, 'one two three');
+  DELETE FROM t1 WHERE rowid=123;
+  SELECT * FROM t1_config
+} {secure-delete 1 version 5}
+
+do_execsql_test 1.11 {
+  INSERT INTO t1(t1) VALUES('rebuild');
+  SELECT * FROM t1_config
+} {secure-delete 1 version 4}
+
+do_execsql_test 1.12 {
+  SELECT * FROM t1_config
+} {secure-delete 1 version 4}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE xyz USING fts5(x);
+  INSERT INTO xyz(rowid, x) VALUES
+      (1, 'one document'),
+      (2, 'two document'),
+      (3, 'three document'),
+      (4, 'four document'),
+      (5, 'five document'),
+      (6, 'six document');
+
+  INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1);
+  SELECT v FROM xyz_config WHERE k='version';
+} {4}
+
+do_execsql_test 2.1 {
+  BEGIN;
+    INSERT INTO xyz(rowid, x) VALUES(7, 'seven document');
+    SAVEPOINT one;
+      DELETE FROM xyz WHERE rowid = 4;
+}
+
+do_execsql_test 2.2 {
+      SELECT v FROM xyz_config WHERE k='version';
+} {5}
+
+do_execsql_test 2.3 {
+    ROLLBACK TO one;
+      SELECT v FROM xyz_config WHERE k='version';
+} {4}
+
+
+do_execsql_test 2.4 {
+      DELETE FROM xyz WHERE rowid = 3;
+  COMMIT;
+  SELECT v FROM xyz_config WHERE k='version';
+} {5}
+
+
 
 
 finish_test
+

ext/misc/base64.c

@@ -148,9 +148,9 @@ static char* toBase64( u8 *pIn, int nbIn, char *pOut ){
 }
 
 /* Skip over text which is not base64 numeral(s). */
-static char * skipNonB64( char *s ){
+static char * skipNonB64( char *s, int nc ){
   char c;
-  while( (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s;
+  while( nc-- > 0 && (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s;
   return s;
 }
 
@@ -159,7 +159,7 @@ static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){
   if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn;
   while( ncIn>0 && *pIn!=PAD_CHAR ){
     static signed char nboi[] = { 0, 0, 1, 2, 3 };
-    char *pUse = skipNonB64(pIn);
+    char *pUse = skipNonB64(pIn, ncIn);
     unsigned long qv = 0L;
     int nti, nbo, nac;
     ncIn -= (pUse - pIn);
@@ -219,9 +219,16 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){
       sqlite3_result_error(context, "blob expanded to base64 too big", -1);
       return;
     }
+    bBuf = (u8*)sqlite3_value_blob(av[0]);
+    if( !bBuf ){
+      if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){
+        goto memFail;
+      }
+      sqlite3_result_text(context,"",-1,SQLITE_STATIC);
+      break;
+    }
     cBuf = sqlite3_malloc(nc);
     if( !cBuf ) goto memFail;
-    bBuf = (u8*)sqlite3_value_blob(av[0]);
     nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf);
     sqlite3_result_text(context, cBuf, nc, sqlite3_free);
     break;
@@ -234,9 +241,16 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){
     }else if( nb<1 ){
       nb = 1;
     }
+    cBuf = (char *)sqlite3_value_text(av[0]);
+    if( !cBuf ){
+      if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){
+        goto memFail;
+      }
+      sqlite3_result_zeroblob(context, 0);
+      break;
+    }
     bBuf = sqlite3_malloc(nb);
     if( !bBuf ) goto memFail;
-    cBuf = (char *)sqlite3_value_text(av[0]);
     nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf);
     sqlite3_result_blob(context, bBuf, nb, sqlite3_free);
     break;

ext/misc/base85.c

@@ -140,9 +140,9 @@ static u8 base85DigitValue( char c ){
 #define B85_DARK_MAX 80
 
 
-static char * skipNonB85( char *s ){
+static char * skipNonB85( char *s, int nc ){
   char c;
-  while( (c = *s) && !IS_B85(c) ) ++s;
+  while( nc-- > 0 && (c = *s) && !IS_B85(c) ) ++s;
   return s;
 }
 
@@ -212,7 +212,7 @@ static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){
   if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn;
   while( ncIn>0 ){
     static signed char nboi[] = { 0, 0, 1, 2, 3, 4 };
-    char *pUse = skipNonB85(pIn);
+    char *pUse = skipNonB85(pIn, ncIn);
     unsigned long qv = 0L;
     int nti, nbo;
     ncIn -= (pUse - pIn);
@@ -297,9 +297,16 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){
       sqlite3_result_error(context, "blob expanded to base85 too big", -1);
       return;
     }
+    bBuf = (u8*)sqlite3_value_blob(av[0]);
+    if( !bBuf ){
+      if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){
+        goto memFail;
+      }
+      sqlite3_result_text(context,"",-1,SQLITE_STATIC);
+      break;
+    }
     cBuf = sqlite3_malloc(nc);
     if( !cBuf ) goto memFail;
-    bBuf = (u8*)sqlite3_value_blob(av[0]);
     nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf);
     sqlite3_result_text(context, cBuf, nc, sqlite3_free);
     break;
@@ -312,9 +319,16 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){
     }else if( nb<1 ){
       nb = 1;
     }
+    cBuf = (char *)sqlite3_value_text(av[0]);
+    if( !cBuf ){
+      if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){
+        goto memFail;
+      }
+      sqlite3_result_zeroblob(context, 0);
+      break;
+    }
     bBuf = sqlite3_malloc(nb);
     if( !bBuf ) goto memFail;
-    cBuf = (char *)sqlite3_value_text(av[0]);
     nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf);
     sqlite3_result_blob(context, bBuf, nb, sqlite3_free);
     break;

ext/misc/randomjson.c

@@ -0,0 +1,202 @@
+/*
+** 2023-04-28
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This SQLite extension implements a the random_json(SEED) and
+** random_json5(SEED) functions.  Given a numeric SEED value, these
+** routines generate pseudo-random JSON or JSON5, respectively.  The
+** same value is always generated for the same seed.
+**
+** These SQL functions are intended for testing.  They do not have any
+** practical real-world use, that we know of.
+**
+** COMPILE:
+**
+**     gcc --shared -fPIC -o randomjson.so -I. ext/misc/randomjson.c
+**
+** USING FROM THE CLI:
+**
+**     .load ./randomjson
+**     SELECT random_json(1);
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+
+/* Pseudo-random number generator */
+typedef struct Prng {
+  unsigned int x, y;
+} Prng;
+
+/* Reseed the PRNG */
+static void prngSeed(Prng *p, unsigned int iSeed){
+  p->x = iSeed | 1;
+  p->y = iSeed;
+}
+
+/* Extract a random number */
+static unsigned int prngInt(Prng *p){
+  p->x = (p->x>>1) ^ ((1+~(p->x&1)) & 0xd0000001);
+  p->y = p->y*1103515245 + 12345;
+  return p->x ^ p->y;
+}
+
+static const char *azJsonAtoms[] = {
+  /* JSON                    /* JSON-5 */
+  "0",                       "0",
+  "1",                       "1",
+  "-1",                      "-1",
+  "2",                       "+2",
+  "3",                       "3",
+  "2.5",                     "2.5",
+  "0.75",                    ".75",
+  "-4.0e2",                  "-4.e2",
+  "5.0e-3",                  "+5e-3",
+  "0",                       "0x0",
+  "512",                     "0x200",
+  "256",                     "+0x100",
+  "-2748",                   "-0xabc",
+  "true",                    "true",
+  "false",                   "false",
+  "null",                    "null",
+  "9.0e999",                 "Infinity",
+  "-9.0e999",                "-Infinity",
+  "9.0e999",                 "+Infinity",
+  "null",                    "NaN",
+  "-0.0005123",              "-0.0005123",
+  "4.35e-3",                 "+4.35e-3",
+  "\"gem\\\"hay\"",          "\"gem\\\"hay\"",
+  "\"icy'joy\"",             "'icy\\'joy\'",
+  "\"keylog\"",              "\"key\\\nlog\"",
+  "\"mix\\\\\\tnet\"",       "\"mix\\\\\\tnet\"",
+  "{}",                      "{}",
+  "[]",                      "[]",
+  "[]",                      "[/*empty*/]",
+  "{}",                      "{//empty\n}",
+  "\"ask\"",                 "\"ask\"",
+  "\"bag\"",                 "\"bag\"",
+  "\"can\"",                 "\"can\"",
+  "\"day\"",                 "\"day\"",
+  "\"end\"",                 "'end'",
+  "\"fly\"",                 "\"fly\"",
+  "\"\"",                    "\"\"",
+};
+static const char *azJsonTemplate[] = {
+  /* JSON                                      JSON-5 */
+  "{\"a\":%,\"b\":%,\"c\":%}",                 "{a:%,b:%,c:%}",
+  "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"e\":%}", "{a:%,b:%,c:%,d:%,e:%}",
+  "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}",  "{a:%,b:%,c:%,d:%,\"\":%}",
+  "{\"d\":%}",                                 "{d:%}",
+  "{\"eeee\":%, \"ffff\":%}",                  "{eeee:% /*and*/, ffff:%}",
+  "{\"$g\":%,\"_h_\":%}",                      "{$g:%,_h_:%,}",
+  "{\"x\":%,\n  \"y\":%}",                     "{\"x\":%,\n  \"y\":%}",
+  "{\"a b c d\":%,\"e\":%,\"f\":%,\"x\":%,\"y\":%}",
+                                           "{\"a b c d\":%,e:%,f:%,x:%,y:%}",
+  "{\"Z\":%}",                                 "{Z:%,}",
+  "[%]",                                       "[%,]",
+  "[%,%]",                                     "[%,%]",
+  "[%,%,%]",                                   "[%,%,%,]",
+  "[%,%,%,%]",                                 "[%,%,%,%]",
+  "[%,%,%,%,%]",                               "[%,%,%,%,%]",
+};
+
+#define count(X)  (sizeof(X)/sizeof(X[0]))
+
+#define STRSZ 10000
+
+static void jsonExpand(
+  const char *zSrc,
+  char *zDest,
+  Prng *p,
+  int eType,            /* 0 for JSON, 1 for JSON5 */
+  unsigned int r        /* Growth probability 0..1000.  0 means no growth */
+){
+  unsigned int i, j, k;
+  const char *z;
+  size_t n;
+
+  j = 0;
+  if( zSrc==0 ){
+    k = prngInt(p)%(count(azJsonTemplate)/2);
+    k = k*2 + eType;
+    zSrc = azJsonTemplate[k];
+  }
+  if( strlen(zSrc)>=STRSZ/10 ) r = 0;
+  for(i=0; zSrc[i]; i++){
+    if( zSrc[i]!='%' ){
+      if( j<STRSZ ) zDest[j++] = zSrc[i];
+      continue;
+    }
+    if( r==0 || (r<1000 && (prngInt(p)%1000)<=r) ){
+      /* Fill in without values without any new % */
+      k = prngInt(p)%(count(azJsonAtoms)/2);
+      k = k*2 + eType;
+      z = azJsonAtoms[k];
+    }else{
+      /* Add new % terms */
+      k = prngInt(p)%(count(azJsonTemplate)/2);
+      k = k*2 + eType;
+      z = azJsonTemplate[k];
+    }
+    n = strlen(z);
+    if( j+n<STRSZ ){
+      memcpy(&zDest[j], z, n);
+      j += n;
+    }
+  }
+  zDest[STRSZ-1] = 0;
+  if( j<STRSZ ) zDest[j] = 0;
+}
+
+static void randJsonFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  unsigned int iSeed;
+  int eType = *(int*)sqlite3_user_data(context);
+  Prng prng;
+  char z1[STRSZ+1], z2[STRSZ+1];
+
+  iSeed = (unsigned int)sqlite3_value_int(argv[0]);
+  prngSeed(&prng, iSeed);
+  jsonExpand(0, z2, &prng, eType, 1000);
+  jsonExpand(z2, z1, &prng, eType, 1000);
+  jsonExpand(z1, z2, &prng, eType, 100);
+  jsonExpand(z2, z1, &prng, eType, 0);
+  sqlite3_result_text(context, z1, -1, SQLITE_TRANSIENT);
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_randomjson_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  static int cOne = 1;
+  static int cZero = 0;
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;  /* Unused parameter */
+  rc = sqlite3_create_function(db, "random_json", 1,
+                   SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC,
+                   &cZero, randJsonFunc, 0, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "random_json5", 1,
+                   SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC,
+                   &cOne, randJsonFunc, 0, 0);
+  }
+  return rc;
+}

ext/misc/series.c

@@ -1,5 +1,5 @@
 /*
-** 2015-08-18
+** 2015-08-18, 2023-04-28
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -12,7 +12,19 @@
 **
 ** This file demonstrates how to create a table-valued-function using
 ** a virtual table.  This demo implements the generate_series() function
-** which gives similar results to the eponymous function in PostgreSQL.
+** which gives the same results as the eponymous function in PostgreSQL,
+** within the limitation that its arguments are signed 64-bit integers.
+**
+** Considering its equivalents to generate_series(start,stop,step): A
+** value V[n] sequence is produced for integer n ascending from 0 where
+**  ( V[n] == start + n * step  &&  sgn(V[n] - stop) * sgn(step) >= 0 )
+** for each produced value (independent of production time ordering.)
+**
+** All parameters must be either integer or convertable to integer.
+** The start parameter is required.
+** The stop parameter defaults to (1<<32)-1 (aka 4294967295 or 0xffffffff)
+** The step parameter defaults to 1 and 0 is treated as 1.
+**
 ** Examples:
 **
 **      SELECT * FROM generate_series(0,100,5);
@@ -28,6 +40,14 @@
 **
 ** Integers 20 through 29.
 **
+**      SELECT * FROM generate_series(0,-100,-5);
+**
+** Integers 0 -5 -10 ... -100.
+**
+**      SELECT * FROM generate_series(0,-1);
+**
+** Empty sequence.
+**
 ** HOW IT WORKS
 **
 ** The generate_series "function" is really a virtual table with the
@@ -40,6 +60,9 @@
 **       step HIDDEN
 **     );
 **
+** The virtual table also has a rowid, logically equivalent to n+1 where
+** "n" is the ascending integer in the aforesaid production definition.
+**
 ** Function arguments in queries against this virtual table are translated
 ** into equality constraints against successive hidden columns.  In other
 ** words, the following pairs of queries are equivalent to each other:
@@ -72,9 +95,96 @@
 SQLITE_EXTENSION_INIT1
 #include <assert.h>
 #include <string.h>
+#include <limits.h>
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
+/*
+** Return that member of a generate_series(...) sequence whose 0-based
+** index is ix. The 0th member is given by smBase. The sequence members
+** progress per ix increment by smStep.
+*/
+static sqlite3_int64 genSeqMember(sqlite3_int64 smBase,
+                                  sqlite3_int64 smStep,
+                                  sqlite3_uint64 ix){
+  if( ix>=(sqlite3_uint64)LLONG_MAX ){
+    /* Get ix into signed i64 range. */
+    ix -= (sqlite3_uint64)LLONG_MAX;
+    smBase += LLONG_MAX * smStep;
+  }
+  return smBase + ((sqlite3_int64)ix)*smStep;
+}
 
+typedef unsigned char u8;
+
+typedef struct SequenceSpec {
+  sqlite3_int64 iBase;         /* Starting value ("start") */
+  sqlite3_int64 iTerm;         /* Given terminal value ("stop") */
+  sqlite3_int64 iStep;         /* Increment ("step") */
+  sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */
+  sqlite3_uint64 uSeqIndexNow; /* Current index during generation */
+  sqlite3_int64 iValueNow;     /* Current value during generation */
+  u8 isNotEOF;                 /* Sequence generation not exhausted */
+  u8 isReversing;              /* Sequence is being reverse generated */
+} SequenceSpec;
+
+/*
+** Prepare a SequenceSpec for use in generating an integer series
+** given initialized iBase, iTerm and iStep values. Sequence is
+** initialized per given isReversing. Other members are computed.
+*/
+void setupSequence( SequenceSpec *pss ){
+  pss->uSeqIndexMax = 0;
+  pss->isNotEOF = 0;
+  if( pss->iTerm < pss->iBase ){
+    sqlite3_uint64 nuspan = (sqlite3_uint64)(pss->iBase-pss->iTerm);
+    if( pss->iStep<0 ){
+      pss->isNotEOF = 1;
+      if( nuspan==ULONG_MAX ){
+        pss->uSeqIndexMax = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1;
+      }else if( pss->iStep>LLONG_MIN ){
+        pss->uSeqIndexMax = nuspan/-pss->iStep;
+      }
+    }
+  }else if( pss->iTerm > pss->iBase ){
+    sqlite3_uint64 puspan = (sqlite3_uint64)(pss->iTerm-pss->iBase);
+    if( pss->iStep>0 ){
+      pss->isNotEOF = 1;
+      pss->uSeqIndexMax = puspan/pss->iStep;
+    }
+  }else if( pss->iTerm == pss->iBase ){
+      pss->isNotEOF = 1;
+      pss->uSeqIndexMax = 0;
+  }
+  pss->uSeqIndexNow = (pss->isReversing)? pss->uSeqIndexMax : 0;
+  pss->iValueNow = (pss->isReversing)
+    ? genSeqMember(pss->iBase, pss->iStep, pss->uSeqIndexMax)
+    : pss->iBase;
+}
+
+/*
+** Progress sequence generator to yield next value, if any.
+** Leave its state to either yield next value or be at EOF.
+** Return whether there is a next value, or 0 at EOF.
+*/
+int progressSequence( SequenceSpec *pss ){
+  if( !pss->isNotEOF ) return 0;
+  if( pss->isReversing ){
+    if( pss->uSeqIndexNow > 0 ){
+      pss->uSeqIndexNow--;
+      pss->iValueNow -= pss->iStep;
+    }else{
+      pss->isNotEOF = 0;
+    }
+  }else{
+    if( pss->uSeqIndexNow < pss->uSeqIndexMax ){
+      pss->uSeqIndexNow++;
+      pss->iValueNow += pss->iStep;
+    }else{
+      pss->isNotEOF = 0;
+    }
+  }
+  return pss->isNotEOF;
+}
 
 /* series_cursor is a subclass of sqlite3_vtab_cursor which will
 ** serve as the underlying representation of a cursor that scans
@@ -83,12 +193,7 @@ SQLITE_EXTENSION_INIT1
 typedef struct series_cursor series_cursor;
 struct series_cursor {
   sqlite3_vtab_cursor base;  /* Base class - must be first */
-  int isDesc;                /* True to count down rather than up */
-  sqlite3_int64 iRowid;      /* The rowid */
-  sqlite3_int64 iValue;      /* Current value ("value") */
-  sqlite3_int64 mnValue;     /* Mimimum value ("start") */
-  sqlite3_int64 mxValue;     /* Maximum value ("stop") */
-  sqlite3_int64 iStep;       /* Increment ("step") */
+  SequenceSpec ss;           /* (this) Derived class data */
 };
 
 /*
@@ -170,12 +275,7 @@ static int seriesClose(sqlite3_vtab_cursor *cur){
 */
 static int seriesNext(sqlite3_vtab_cursor *cur){
   series_cursor *pCur = (series_cursor*)cur;
-  if( pCur->isDesc ){
-    pCur->iValue -= pCur->iStep;
-  }else{
-    pCur->iValue += pCur->iStep;
-  }
-  pCur->iRowid++;
+  progressSequence( & pCur->ss );
   return SQLITE_OK;
 }
 
@@ -191,10 +291,10 @@ static int seriesColumn(
   series_cursor *pCur = (series_cursor*)cur;
   sqlite3_int64 x = 0;
   switch( i ){
-    case SERIES_COLUMN_START:  x = pCur->mnValue; break;
-    case SERIES_COLUMN_STOP:   x = pCur->mxValue; break;
-    case SERIES_COLUMN_STEP:   x = pCur->iStep;   break;
-    default:                   x = pCur->iValue;  break;
+    case SERIES_COLUMN_START:  x = pCur->ss.iBase; break;
+    case SERIES_COLUMN_STOP:   x = pCur->ss.iTerm; break;
+    case SERIES_COLUMN_STEP:   x = pCur->ss.iStep;   break;
+    default:                   x = pCur->ss.iValueNow;  break;
   }
   sqlite3_result_int64(ctx, x);
   return SQLITE_OK;
@@ -207,7 +307,7 @@ static int seriesColumn(
 */
 static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
   series_cursor *pCur = (series_cursor*)cur;
-  *pRowid = pCur->iRowid;
+  *pRowid = ((sqlite3_int64)pCur->ss.uSeqIndexNow + 1);
   return SQLITE_OK;
 }
 
@@ -217,14 +317,10 @@ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
 */
 static int seriesEof(sqlite3_vtab_cursor *cur){
   series_cursor *pCur = (series_cursor*)cur;
-  if( pCur->isDesc ){
-    return pCur->iValue < pCur->mnValue;
-  }else{
-    return pCur->iValue > pCur->mxValue;
-  }
+  return !pCur->ss.isNotEOF;
 }
 
-/* True to cause run-time checking of the start=, stop=, and/or step= 
+/* True to cause run-time checking of the start=, stop=, and/or step=
 ** parameters.  The only reason to do this is for testing the
 ** constraint checking logic for virtual tables in the SQLite core.
 */
@@ -235,7 +331,7 @@ static int seriesEof(sqlite3_vtab_cursor *cur){
 /*
 ** This method is called to "rewind" the series_cursor object back
 ** to the first row of output.  This method is always called at least
-** once prior to any call to seriesColumn() or seriesRowid() or 
+** once prior to any call to seriesColumn() or seriesRowid() or
 ** seriesEof().
 **
 ** The query plan selected by seriesBestIndex is passed in the idxNum
@@ -255,7 +351,7 @@ static int seriesEof(sqlite3_vtab_cursor *cur){
 ** (so that seriesEof() will return true) if the table is empty.
 */
 static int seriesFilter(
-  sqlite3_vtab_cursor *pVtabCursor, 
+  sqlite3_vtab_cursor *pVtabCursor,
   int idxNum, const char *idxStrUnused,
   int argc, sqlite3_value **argv
 ){
@@ -263,46 +359,41 @@ static int seriesFilter(
   int i = 0;
   (void)idxStrUnused;
   if( idxNum & 1 ){
-    pCur->mnValue = sqlite3_value_int64(argv[i++]);
+    pCur->ss.iBase = sqlite3_value_int64(argv[i++]);
   }else{
-    pCur->mnValue = 0;
+    pCur->ss.iBase = 0;
   }
   if( idxNum & 2 ){
-    pCur->mxValue = sqlite3_value_int64(argv[i++]);
+    pCur->ss.iTerm = sqlite3_value_int64(argv[i++]);
   }else{
-    pCur->mxValue = 0xffffffff;
+    pCur->ss.iTerm = 0xffffffff;
   }
   if( idxNum & 4 ){
-    pCur->iStep = sqlite3_value_int64(argv[i++]);
-    if( pCur->iStep==0 ){
-      pCur->iStep = 1;
-    }else if( pCur->iStep<0 ){
-      pCur->iStep = -pCur->iStep;
+    pCur->ss.iStep = sqlite3_value_int64(argv[i++]);
+    if( pCur->ss.iStep==0 ){
+      pCur->ss.iStep = 1;
+    }else if( pCur->ss.iStep<0 ){
       if( (idxNum & 16)==0 ) idxNum |= 8;
     }
   }else{
-    pCur->iStep = 1;
+    pCur->ss.iStep = 1;
   }
   for(i=0; i<argc; i++){
     if( sqlite3_value_type(argv[i])==SQLITE_NULL ){
       /* If any of the constraints have a NULL value, then return no rows.
       ** See ticket https://www.sqlite.org/src/info/fac496b61722daf2 */
-      pCur->mnValue = 1;
-      pCur->mxValue = 0;
+      pCur->ss.iBase = 1;
+      pCur->ss.iTerm = 0;
+      pCur->ss.iStep = 1;
       break;
     }
   }
   if( idxNum & 8 ){
-    pCur->isDesc = 1;
-    pCur->iValue = pCur->mxValue;
-    if( pCur->iStep>0 ){
-      pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep;
-    }
+    pCur->ss.isReversing = pCur->ss.iStep > 0;
   }else{
-    pCur->isDesc = 0;
-    pCur->iValue = pCur->mnValue;
+    pCur->ss.isReversing = pCur->ss.iStep < 0;
   }
-  pCur->iRowid = 1;
+  setupSequence( &pCur->ss );
   return SQLITE_OK;
 }
 

ext/misc/shathree.c

@@ -10,7 +10,8 @@
 **
 ******************************************************************************
 **
-** This SQLite extension implements functions that compute SHA3 hashes.
+** This SQLite extension implements functions that compute SHA3 hashes
+** in the way described by the (U.S.) NIST FIPS 202 SHA-3 Standard.
 ** Two SQL functions are implemented:
 **
 **     sha3(X,SIZE)

ext/misc/zipfile.c

@@ -1537,9 +1537,19 @@ static u32 zipfileGetTime(sqlite3_value *pVal){
 */
 static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){
   if( pOld ){
-    ZipfileEntry **pp;
-    for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext));
-    *pp = (*pp)->pNext;
+    if( pTab->pFirstEntry==pOld ){
+      pTab->pFirstEntry = pOld->pNext;
+      if( pTab->pLastEntry==pOld ) pTab->pLastEntry = 0;
+    }else{
+      ZipfileEntry *p;
+      for(p=pTab->pFirstEntry; p; p=p->pNext){
+        if( p->pNext==pOld ){
+          p->pNext = pOld->pNext;
+          if( pTab->pLastEntry==pOld ) pTab->pLastEntry = p;
+          break;
+        }
+      }
+    }
     zipfileEntryFree(pOld);
   }
 }

ext/rbu/rbuexlock.test

@@ -207,5 +207,86 @@ do_test 3.4.0 {
 } SQLITE_OK
 rbu close
 
+#-------------------------------------------------------------------------
+reset_db
+forcedelete rbu1.db
+forcedelete rbu2.db
+
+sqlite3 rbu rbu1.db 
+do_execsql_test -db rbu 4.1 {
+  CREATE TABLE data_t1(a, b, rbu_control);
+  INSERT INTO data_t1 VALUES(1, 'one', 0);
+}
+rbu close
+sqlite3 rbu rbu2.db 
+do_execsql_test -db rbu 4.2 {
+  CREATE TABLE data_t1(a, b, rbu_control);
+  INSERT INTO data_t1 VALUES(2, 'two', 0);
+}
+rbu close
+
+do_execsql_test 4.3 {
+  CREATE TABLE t1(a PRIMARY KEY, b);
+}
+db close
+
+do_test 4.4 {
+  sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu1.db
+  rbu step
+  rbu state
+} {oal}
+
+sqlite3 cons test.db
+do_execsql_test -db cons 4.5 {
+  SELECT * FROM t1
+} {}
+
+do_test 4.6 { rbu step ; rbu state } {oal}
+do_test 4.7 { rbu step ; rbu state } {move}
+do_execsql_test -db cons 4.8 {
+  SELECT * FROM t1
+} {}
+do_test 4.9 { rbu step ; rbu state } {checkpoint}
+do_test 4.10 {
+  catchsql { SELECT * FROM t1 } cons
+} {1 {database is locked}}
+do_test 4.11 { rbu step ; rbu state } {checkpoint}
+do_test 4.11 { rbu step ; rbu state } {done}
+rbu close
+
+do_test 4.12 {
+  catchsql { SELECT * FROM t1 } cons
+} {0 {1 one}}
+
+do_test 4.13 {
+  sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu2.db
+  rbu step
+  rbu state
+} {oal}
+
+do_test 4.14 {
+  catchsql { SELECT * FROM t1 } cons
+} {0 {1 one}}
+
+do_test 4.15 { rbu step ; rbu state } {oal}
+do_test 4.16 { rbu step ; rbu state } {move}
+
+do_test 4.17 {
+  catchsql { SELECT * FROM t1 } cons
+} {0 {1 one}}
+
+do_test 4.18 { rbu step ; rbu state } {checkpoint}
+do_test 4.19 {
+  catchsql { SELECT * FROM t1 } cons
+} {1 {database is locked}}
+do_test 4.20 { rbu step ; rbu state } {checkpoint}
+do_test 4.21 { rbu step ; rbu state } {done}
+rbu close
+
+do_test 4.22 {
+  catchsql { SELECT * FROM t1 } cons
+} {0 {1 one 2 two}}
+
+cons close
 
 finish_test

ext/rbu/sqlite3rbu.c

@@ -3141,6 +3141,11 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){
   p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
 }
 
+/*
+** This value is copied from the definition of ZIPVFS_CTRL_FILE_POINTER
+** in zipvfs.h. 
+*/
+#define RBU_ZIPVFS_CTRL_FILE_POINTER 230439
 
 /*
 ** Take an EXCLUSIVE lock on the database file. Return SQLITE_OK if
@@ -3149,7 +3154,11 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){
 static int rbuLockDatabase(sqlite3 *db){
   int rc = SQLITE_OK;
   sqlite3_file *fd = 0;
-  sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd);
+
+  sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd);
+  if( fd==0 ){
+    sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd);
+  }
 
   if( fd->pMethods ){
     rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED);

ext/recover/recover1.test

@@ -316,5 +316,18 @@ do_execsql_test 16.9 {
   COMMIT;
 } {1 2 3 4}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 17.1 {
+  CREATE TABLE t(a, PRIMARY KEY(a, a COLLATE NOCASE)) WITHOUT ROWID;
+  INSERT INTO t VALUES('abc');
+  INSERT INTO t VALUES('def');
+}
+do_test 17.2 {
+  set R [sqlite3_recover_init db main test.db2]
+  $R run
+  list [catch { $R finish } msg] $msg
+} {0 {}}
+
 finish_test
 

ext/recover/recoverbuild.test

@@ -28,7 +28,7 @@ do_execsql_test 1.0 {
 forcedelete test.db2
 do_test 1.1 {
   set R [sqlite3_recover_init db main test.db2]
-} {sqlite_recover1}
+} {/sqlite_recover.*/}
 
 do_test 1.2 {
   $R run

ext/recover/sqlite3recover.c

@@ -1106,7 +1106,7 @@ static void recoverAddTable(
       int iField = sqlite3_column_int(pStmt, 0);
       int iCol = sqlite3_column_int(pStmt, 1);
 
-      assert( iField<pNew->nCol && iCol<pNew->nCol );
+      assert( iCol<pNew->nCol );
       pNew->aCol[iCol].iField = iField;
 
       pNew->bIntkey = 0;

ext/rtree/rtree.c

@@ -471,16 +471,17 @@ struct RtreeMatchArg {
 ** at run-time.
 */
 #ifndef SQLITE_BYTEORDER
-#if defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
-    defined(__x86_64) || defined(__x86_64__) || defined(_M_X64)  ||    \
-    defined(_M_AMD64) || defined(_M_ARM)     || defined(__x86)   ||    \
-    defined(__arm__)
-# define SQLITE_BYTEORDER    1234
-#elif defined(sparc)    || defined(__ppc__)
-# define SQLITE_BYTEORDER    4321
-#else
-# define SQLITE_BYTEORDER    0     /* 0 means "unknown at compile-time" */
-#endif
+# if defined(i386)      || defined(__i386__)      || defined(_M_IX86) ||    \
+     defined(__x86_64)  || defined(__x86_64__)    || defined(_M_X64)  ||    \
+     defined(_M_AMD64)  || defined(_M_ARM)        || defined(__x86)   ||    \
+     defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64)
+#   define SQLITE_BYTEORDER    1234
+# elif defined(sparc)     || defined(__ppc__) || \
+       defined(__ARMEB__) || defined(__AARCH64EB__)
+#   define SQLITE_BYTEORDER    4321
+# else
+#   define SQLITE_BYTEORDER 0
+# endif
 #endif
 
 

ext/session/sessionD.test

@@ -41,30 +41,6 @@ proc scksum {db dbname} {
   return [md5 $txt]
 }
 
-proc do_diff_test {tn setup} {
-  reset_db
-  forcedelete test.db2
-  execsql { ATTACH 'test.db2' AS aux }
-  execsql $setup
-
-  sqlite3session S db main
-  foreach tbl [db eval {SELECT name FROM sqlite_master WHERE type='table'}] {
-    S attach $tbl
-    S diff aux $tbl
-  }
-
-  set C [S changeset]
-  S delete
-
-  sqlite3 db2 test.db2
-  sqlite3changeset_apply db2 $C ""
-  uplevel do_test $tn.1 [list {execsql { PRAGMA integrity_check } db2}] ok
-  db2 close
-
-  set cksum [scksum db main]
-  uplevel do_test $tn.2 [list {scksum db aux}] [list $cksum]
-}
-
 # Ensure that the diff produced by comparing the current contents of [db]
 # with itself is empty.
 proc do_empty_diff_test {tn} {

ext/session/session_common.tcl

@@ -52,6 +52,7 @@ proc do_conflict_test {tn args} {
   proc bgerror {args} { set ::background_error $args }
 
   sqlite3session S db main
+  S object_config rowid 1
   foreach t $O(-tables) { S attach $t }
   execsql $O(-sql)
 
@@ -81,6 +82,7 @@ proc changeset_from_sql {sql {dbname main}} {
   }
   set rc [catch {
     sqlite3session S db $dbname
+    S object_config rowid 1
     db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" {
       S attach $name
     }
@@ -138,6 +140,7 @@ proc do_then_apply_sql {args} {
   proc xConflict args { incr ::n_conflict ; return "OMIT" }
   set rc [catch {
     sqlite3session S db $dbname
+    S object_config rowid 1
     db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" {
       S attach $name
     }
@@ -162,6 +165,8 @@ proc do_then_apply_sql {args} {
 
 proc do_iterator_test {tn tbl_list sql res} {
   sqlite3session S db main
+  S object_config rowid 1
+
   if {[llength $tbl_list]==0} { S attach * }
   foreach t $tbl_list {S attach $t}
 
@@ -171,6 +176,7 @@ proc do_iterator_test {tn tbl_list sql res} {
   foreach v $res { lappend r $v }
 
   set x [list]
+# set ::c [S changeset] ; execsql_pp { SELECT quote($::c) }
   sqlite3session_foreach c [S changeset] { lappend x $c }
   uplevel do_test $tn [list [list set {} $x]] [list $r]
 
@@ -245,3 +251,49 @@ proc number_name {n} {
   if {$txt==""} {set txt zero}
   return $txt
 }
+
+proc scksum {db dbname} {
+
+  if {$dbname=="temp"} {
+    set master sqlite_temp_master
+  } else {
+    set master $dbname.sqlite_master
+  }
+
+  set alltab [$db eval "SELECT name FROM $master WHERE type='table'"]
+  set txt [$db eval "SELECT * FROM $master ORDER BY type,name,sql"]
+  foreach tab $alltab {
+    set cols [list]
+    db eval "PRAGMA $dbname.table_info = $tab" x { 
+      lappend cols "quote($x(name))" 
+    }
+    set cols [join $cols ,]
+    append txt [db eval "SELECT $cols FROM $dbname.$tab ORDER BY $cols"]
+  }
+  return [md5 $txt]
+}
+
+proc do_diff_test {tn setup} {
+  reset_db
+  forcedelete test.db2
+  execsql { ATTACH 'test.db2' AS aux }
+  execsql $setup
+
+  sqlite3session S db main
+  S object_config rowid 1
+  foreach tbl [db eval {SELECT name FROM sqlite_master WHERE type='table'}] {
+    S attach $tbl
+    S diff aux $tbl
+  }
+
+  set C [S changeset]
+  S delete
+
+  sqlite3 db2 test.db2
+  sqlite3changeset_apply db2 $C ""
+  uplevel do_test $tn.1 [list {execsql { PRAGMA integrity_check } db2}] ok
+  db2 close
+
+  set cksum [scksum db main]
+  uplevel do_test $tn.2 [list {scksum db aux}] [list $cksum]
+}

ext/session/sessionrebase.test

@@ -84,6 +84,7 @@ proc do_rebase_test {tn sql1 sql2 conflict_handler {testsql ""} {testres ""}} {
     db eval BEGIN
 
     sqlite3session S1 db main
+    S1 object_config rowid 1
     S1 attach *
     execsql $sql1 db
     set c1 [S1 changeset]
@@ -91,6 +92,7 @@ proc do_rebase_test {tn sql1 sql2 conflict_handler {testsql ""} {testres ""}} {
 
     if {$i==1} {
       sqlite3session S2 db2 main
+      S2 object_config rowid 1
       S2 attach *
       execsql $sql2 db2
       set c2 [S2 changeset]
@@ -100,6 +102,7 @@ proc do_rebase_test {tn sql1 sql2 conflict_handler {testsql ""} {testres ""}} {
       foreach sql [split $sql2 ";"] {
         if {[string is space $sql]} continue
         sqlite3session S2 db2 main
+        S2 object_config rowid 1
         S2 attach *
         execsql $sql db2
         lappend c2 [S2 changeset]
@@ -341,6 +344,79 @@ do_rebase_test 2.2.3 {
   OMIT
 } { SELECT * FROM t2 WHERE z='B' } { 1 one B }
 
+
+reset_db
+do_execsql_test 2.3.0 {
+  CREATE TABLE t1 (b TEXT);
+  INSERT INTO t1(rowid, b) VALUES(1, 'one');
+  INSERT INTO t1(rowid, b) VALUES(2, 'two');
+  INSERT INTO t1(rowid, b) VALUES(3, 'three');
+}
+do_rebase_test 2.3.1 {
+  UPDATE t1 SET b = 'two.1' WHERE rowid=2
+} {
+  UPDATE t1 SET b = 'two.2' WHERE rowid=2;
+} {
+  OMIT
+} { SELECT rowid, * FROM t1 } {1 one 2 two.1 3 three}
+
+do_rebase_test 2.3.2 {
+  UPDATE t1 SET b = 'two.1' WHERE rowid=2
+} {
+  UPDATE t1 SET b = 'two.2' WHERE rowid=2;
+} {
+  REPLACE
+} { SELECT rowid, * FROM t1 } {1 one 2 two.2 3 three}
+
+do_rebase_test 2.3.3 {
+  DELETE FROM t1 WHERE rowid=3
+} {
+  DELETE FROM t1 WHERE rowid=3;
+} {
+  OMIT
+} { SELECT rowid, * FROM t1 } {1 one 2 two}
+
+do_rebase_test 2.3.4 {
+  DELETE FROM t1 WHERE rowid=1
+} {
+  UPDATE t1 SET b='one.2' WHERE rowid=1
+} {
+  OMIT
+} { SELECT rowid, * FROM t1 } {2 two 3 three}
+
+do_rebase_test 2.3.6 {
+  UPDATE t1 SET b='three.1' WHERE rowid=3
+} {
+  DELETE FROM t1 WHERE rowid=3;
+} {
+  OMIT
+} { SELECT rowid, * FROM t1 } {1 one 2 two 3 three.1}
+
+do_rebase_test 2.3.7 {
+  UPDATE t1 SET b='three.1' WHERE rowid=3
+} {
+  DELETE FROM t1 WHERE rowid=3;
+} {
+  REPLACE
+} { SELECT rowid, * FROM t1 } {1 one 2 two}
+
+do_rebase_test 2.3.8 {
+  INSERT INTO t1(rowid, b) VALUES(4, 'four.1')
+} {
+  INSERT INTO t1(rowid, b) VALUES(4, 'four.2');
+} {
+  REPLACE
+} { SELECT rowid, * FROM t1 } {1 one 2 two 3 three 4 four.2}
+
+do_rebase_test 2.3.9 {
+  INSERT INTO t1(rowid, b) VALUES(4, 'four.1')
+} {
+  INSERT INTO t1(rowid, b) VALUES(4, 'four.2');
+} {
+  OMIT
+} { SELECT rowid, * FROM t1 } {1 one 2 two 3 three 4 four.1}
+
+
 #-------------------------------------------------------------------------
 reset_db
 do_execsql_test 3.0 {

ext/session/sessionrowid.test

@@ -0,0 +1,281 @@
+# 2011 Mar 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the session module.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source [file join [file dirname [info script]] session_common.tcl]
+source $testdir/tester.tcl
+ifcapable !session {finish_test; return}
+
+set testprefix sessionrowid
+
+do_execsql_test 0.0 { 
+  CREATE TABLE t1(a, b);
+}
+
+foreach {tn rowid bEmpty} {
+  1 0    1
+  2 1    0
+  3 -1   1
+} {
+  do_test 0.$tn {
+    sqlite3session S db main
+    if {$rowid>=0} { S object_config rowid $rowid }
+    S attach t1
+    execsql { INSERT INTO t1 VALUES(1, 2); }
+    expr [string length [S changeset]]==0
+  } $bEmpty
+  S delete
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 1.0 { 
+  CREATE TABLE t1(a, b);
+}
+
+do_iterator_test 1.1 t1 {
+  INSERT INTO t1 VALUES('i', 'one');
+} {
+  {INSERT t1 0 X.. {} {i 1 t i t one}}
+}
+
+do_execsql_test 1.2 {
+  SELECT rowid, * FROM t1
+} {1 i one}
+
+do_iterator_test 1.3 t1 {
+  UPDATE t1 SET b='two'
+} {
+  {UPDATE t1 0 X.. {i 1 {} {} t one} {{} {} {} {} t two}}
+}
+
+do_iterator_test 1.4 t1 {
+  DELETE FROM t1;
+} {
+  {DELETE t1 0 X.. {i 1 t i t two} {}}
+}
+
+do_iterator_test 1.5 t1 {
+  INSERT INTO t1(rowid, a, b) VALUES(14, 'hello', 'world');
+  INSERT INTO t1(rowid, a, b) VALUES(NULL, 'yes', 'no');
+  INSERT INTO t1(rowid, a, b) VALUES(-123, 'ii', 'iii');
+} {
+  {INSERT t1 0 X.. {} {i -123 t ii t iii}}
+  {INSERT t1 0 X.. {} {i 15 t yes t no}}
+  {INSERT t1 0 X.. {} {i 14 t hello t world}}
+}
+
+do_iterator_test 1.6 t1 {
+  UPDATE t1 SET a='deluxe' WHERE rowid=14;
+  DELETE FROM t1 WHERE rowid=-123;
+  INSERT INTO t1 VALUES('x', 'xi');
+} {
+  {DELETE t1 0 X.. {i -123 t ii t iii} {}}
+  {UPDATE t1 0 X.. {i 14 t hello {} {}} {{} {} t deluxe {} {}}}
+  {INSERT t1 0 X.. {} {i 16 t x t xi}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+forcedelete test.db2
+sqlite3 db2 test.db2
+
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a, b);
+}
+do_execsql_test -db db2 2.0.1 {
+  CREATE TABLE t1(a, b);
+}
+
+proc xConflict {args} { 
+  puts "CONFLICT!"
+  return "OMIT" 
+}
+
+do_test 2.1 {
+  set C [changeset_from_sql {
+    INSERT INTO t1 VALUES('abc', 'def');
+  }]
+  sqlite3changeset_apply db2 $C xConflict
+  execsql { SELECT * FROM t1 } db2
+} {abc def}
+do_test 2.2 {
+  set C [changeset_from_sql {
+    UPDATE t1 SET b='hello'
+  }]
+  sqlite3changeset_apply db2 $C xConflict
+  execsql { SELECT * FROM t1 } db2
+} {abc hello}
+do_test 2.3 {
+  set C [changeset_from_sql {
+    DELETE FROM t1 WHERE b='hello'
+  }]
+  sqlite3changeset_apply db2 $C xConflict
+  execsql { SELECT * FROM t1 } db2
+} {}
+
+do_test 2.4 {
+  do_then_apply_sql {
+    INSERT INTO t1 VALUES('i', 'one');
+    INSERT INTO t1 VALUES('ii', 'two');
+    INSERT INTO t1 VALUES('iii', 'three');
+    INSERT INTO t1 VALUES('iv', 'four');
+  }
+  compare_db db db2
+} {}
+
+do_test 2.5 {
+  do_then_apply_sql {
+    DELETE FROM t1 WHERE a='ii';
+    UPDATE t1 SET b='THREE' WHERE a='iii';
+    UPDATE t1 SET a='III' WHERE a='iii';
+    INSERT INTO t1 VALUES('v', 'five');
+  }
+  compare_db db db2
+} {}
+
+do_execsql_test 2.6 {SELECT * FROM t1} {i one III THREE iv four v five}
+do_execsql_test -db db2 2.7 {SELECT * FROM t1} {i one III THREE iv four v five}
+
+#-------------------------------------------------------------------------
+db2 close
+reset_db
+forcedelete test.db2
+sqlite3 db2 test.db2
+
+set init_sql {
+  CREATE TABlE t4(a, b);
+  CREATE INDEX t4a ON t4(a);
+  CREATE UNIQUE INDEX t4b ON t4(b);
+}
+
+do_execsql_test 3.0 $init_sql
+do_execsql_test -db db2 3.0a $init_sql
+
+do_execsql_test -db db2 3.1 {
+  INSERT INTO t4(rowid, a, b) VALUES(43, 'hello', 'world');
+}
+do_conflict_test 3.2 -sql {
+  INSERT INTO t4(rowid, a, b) VALUES(43, 'abc', 'def');
+} -tables t4 -conflicts {
+  {INSERT t4 CONFLICT {i 43 t abc t def} {i 43 t hello t world}}
+}
+do_execsql_test -db db2 3.3 {
+  SELECT * FROM t4
+} {hello world}
+
+do_execsql_test 3.4 { DELETE FROM t4 }
+do_conflict_test 3.5 -sql {
+  INSERT INTO t4(rowid, a, b) VALUES(43, 'abc', 'def');
+} -tables t4 -conflicts {
+  {INSERT t4 CONFLICT {i 43 t abc t def} {i 43 t hello t world}}
+} -policy REPLACE
+do_execsql_test -db db2 3.6 {
+  SELECT * FROM t4
+} {abc def}
+
+do_execsql_test 3.7 { DELETE FROM t4 }
+do_conflict_test 3.8 -sql {
+  INSERT INTO t4(rowid, a, b) VALUES(45, 'xyz', 'def');
+} -tables t4 -conflicts {
+  {INSERT t4 CONSTRAINT {i 45 t xyz t def}}
+} 
+do_execsql_test -db db2 3.9 {
+  SELECT * FROM t4
+} {abc def}
+
+
+do_execsql_test -db db  3.10a { DELETE FROM t4 }
+do_execsql_test -db db2 3.10b { DELETE FROM t4 }
+
+do_execsql_test -db db 3.11a {
+  INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'one');
+  INSERT INTO t4(rowid, a, b) VALUES(222, 'two', 'two');
+}
+do_execsql_test -db db2 3.11b {
+  INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'blip');
+}
+
+do_conflict_test 3.12 -sql {
+  DELETE FROM t4 WHERE a='one';
+} -tables t4 -conflicts {
+  {DELETE t4 DATA {i 111 t one t one} {i 111 t one t blip}}
+} 
+do_execsql_test -db db2 3.13 {
+  SELECT * FROM t4
+} {one blip}
+
+do_conflict_test 3.14 -sql {
+  DELETE FROM t4 WHERE a='two';
+} -tables t4 -conflicts {
+  {DELETE t4 NOTFOUND {i 222 t two t two}}
+} 
+do_execsql_test -db db2 3.15 {
+  SELECT * FROM t4
+} {one blip}
+
+do_execsql_test -db db  3.16a { DELETE FROM t4 }
+do_execsql_test -db db2 3.16b { DELETE FROM t4 }
+
+do_execsql_test -db db 3.17a {
+  INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'one');
+  INSERT INTO t4(rowid, a, b) VALUES(222, 'two', 'two');
+}
+do_execsql_test -db db2 3.17b {
+  INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'blip');
+}
+
+do_conflict_test 3.18 -sql {
+  UPDATE t4 SET b='xyz' WHERE a='one'
+} -tables t4 -conflicts {
+  {UPDATE t4 DATA {i 111 {} {} t one} {{} {} {} {} t xyz} {i 111 t one t blip}}
+} 
+do_execsql_test -db db2 3.19 {
+  SELECT * FROM t4
+} {one blip}
+
+do_conflict_test 3.20 -sql {
+  UPDATE t4 SET b='123' WHERE a='two'
+} -tables t4 -conflicts {
+  {UPDATE t4 NOTFOUND {i 222 {} {} t two} {{} {} {} {} t 123}}
+} 
+do_execsql_test -db db2 3.21 {
+  SELECT * FROM t4
+} {one blip}
+
+#--------------------------------------------------------------------------
+breakpoint
+do_diff_test 4.0 {
+  CREATE TABLE t1(x, y);
+  CREATE TABLE aux.t1(x, y);
+  INSERT INTO t1 VALUES(1, 2);
+}
+
+do_diff_test 4.1 {
+  CREATE TABLE t1(x, y);
+  CREATE TABLE aux.t1(x, y);
+  INSERT INTO aux.t1 VALUES(1, 2);
+}
+
+do_diff_test 4.2 {
+  CREATE TABLE t1(x, y);
+  CREATE TABLE aux.t1(x, y);
+  INSERT INTO t1(rowid, x, y) VALUES(413, 'hello', 'there');
+  INSERT INTO aux.t1(rowid, x, y) VALUES(413, 'hello', 'world');
+}
+
+finish_test
+

ext/session/sessionsize.test

@@ -113,17 +113,17 @@ do_execsql_test 3.0 {
 
 do_test 3.1 {
   sqlite3session S db main
-  S object_config_size -1
+  S object_config size -1
 } 1
 
-do_test 3.2.1 { S object_config_size 0  } 0
-do_test 3.2.2 { S object_config_size -1 } 0
-do_test 3.2.3 { S object_config_size 1  } 1
-do_test 3.2.4 { S object_config_size -1 } 1
+do_test 3.2.1 { S object_config size 0  } 0
+do_test 3.2.2 { S object_config size -1 } 0
+do_test 3.2.3 { S object_config size 1  } 1
+do_test 3.2.4 { S object_config size -1 } 1
 
 do_test 3.3 { S attach t1 } {}
-do_test 3.4 { S object_config_size 1  } {SQLITE_MISUSE}
-do_test 3.4 { S object_config_size -1 } {1}
+do_test 3.4 { S object_config size 1  } {SQLITE_MISUSE}
+do_test 3.4 { S object_config size -1 } {1}
 
 S delete
 

ext/session/sqlite3session.c

@@ -25,6 +25,8 @@ typedef struct SessionInput SessionInput;
 # endif
 #endif
 
+#define SESSIONS_ROWID "_rowid_"
+
 static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE;
 
 typedef struct SessionHook SessionHook;
@@ -46,6 +48,7 @@ struct sqlite3_session {
   int bEnable;                    /* True if currently recording */
   int bIndirect;                  /* True if all changes are indirect */
   int bAutoAttach;                /* True to auto-attach tables */
+  int bImplicitPK;                /* True to handle tables with implicit PK */
   int rc;                         /* Non-zero if an error has occurred */
   void *pFilterCtx;               /* First argument to pass to xTableFilter */
   int (*xTableFilter)(void *pCtx, const char *zTab);
@@ -122,6 +125,7 @@ struct SessionTable {
   char *zName;                    /* Local name of table */
   int nCol;                       /* Number of columns in table zName */
   int bStat1;                     /* True if this is sqlite_stat1 */
+  int bRowid;                     /* True if this table uses rowid for PK */
   const char **azCol;             /* Column names */
   u8 *abPK;                       /* Array of primary key flags */
   int nEntry;                     /* Total number of entries in hash table */
@@ -514,6 +518,7 @@ static unsigned int sessionHashAppendType(unsigned int h, int eType){
 */
 static int sessionPreupdateHash(
   sqlite3_session *pSession,      /* Session object that owns pTab */
+  i64 iRowid,
   SessionTable *pTab,             /* Session table handle */
   int bNew,                       /* True to hash the new.* PK */
   int *piHash,                    /* OUT: Hash value */
@@ -522,48 +527,53 @@ static int sessionPreupdateHash(
   unsigned int h = 0;             /* Hash value to return */
   int i;                          /* Used to iterate through columns */
 
-  assert( *pbNullPK==0 );
-  assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) );
-  for(i=0; i<pTab->nCol; i++){
-    if( pTab->abPK[i] ){
-      int rc;
-      int eType;
-      sqlite3_value *pVal;
+  if( pTab->bRowid ){
+    assert( pTab->nCol-1==pSession->hook.xCount(pSession->hook.pCtx) );
+    h = sessionHashAppendI64(h, iRowid);
+  }else{
+    assert( *pbNullPK==0 );
+    assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) );
+    for(i=0; i<pTab->nCol; i++){
+      if( pTab->abPK[i] ){
+        int rc;
+        int eType;
+        sqlite3_value *pVal;
 
-      if( bNew ){
-        rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal);
-      }else{
-        rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal);
-      }
-      if( rc!=SQLITE_OK ) return rc;
+        if( bNew ){
+          rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal);
+        }else{
+          rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal);
+        }
+        if( rc!=SQLITE_OK ) return rc;
 
-      eType = sqlite3_value_type(pVal);
-      h = sessionHashAppendType(h, eType);
-      if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
-        i64 iVal;
-        if( eType==SQLITE_INTEGER ){
-          iVal = sqlite3_value_int64(pVal);
+        eType = sqlite3_value_type(pVal);
+        h = sessionHashAppendType(h, eType);
+        if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
+          i64 iVal;
+          if( eType==SQLITE_INTEGER ){
+            iVal = sqlite3_value_int64(pVal);
+          }else{
+            double rVal = sqlite3_value_double(pVal);
+            assert( sizeof(iVal)==8 && sizeof(rVal)==8 );
+            memcpy(&iVal, &rVal, 8);
+          }
+          h = sessionHashAppendI64(h, iVal);
+        }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
+          const u8 *z;
+          int n;
+          if( eType==SQLITE_TEXT ){
+            z = (const u8 *)sqlite3_value_text(pVal);
+          }else{
+            z = (const u8 *)sqlite3_value_blob(pVal);
+          }
+          n = sqlite3_value_bytes(pVal);
+          if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM;
+          h = sessionHashAppendBlob(h, n, z);
         }else{
-          double rVal = sqlite3_value_double(pVal);
-          assert( sizeof(iVal)==8 && sizeof(rVal)==8 );
-          memcpy(&iVal, &rVal, 8);
+          assert( eType==SQLITE_NULL );
+          assert( pTab->bStat1==0 || i!=1 );
+          *pbNullPK = 1;
         }
-        h = sessionHashAppendI64(h, iVal);
-      }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
-        const u8 *z;
-        int n;
-        if( eType==SQLITE_TEXT ){
-          z = (const u8 *)sqlite3_value_text(pVal);
-        }else{
-          z = (const u8 *)sqlite3_value_blob(pVal);
-        }
-        n = sqlite3_value_bytes(pVal);
-        if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM;
-        h = sessionHashAppendBlob(h, n, z);
-      }else{
-        assert( eType==SQLITE_NULL );
-        assert( pTab->bStat1==0 || i!=1 );
-        *pbNullPK = 1;
       }
     }
   }
@@ -846,6 +856,7 @@ static int sessionMergeUpdate(
 */
 static int sessionPreupdateEqual(
   sqlite3_session *pSession,      /* Session object that owns SessionTable */
+  i64 iRowid,                     /* Rowid value if pTab->bRowid */
   SessionTable *pTab,             /* Table associated with change */
   SessionChange *pChange,         /* Change to compare to */
   int op                          /* Current pre-update operation */
@@ -853,6 +864,11 @@ static int sessionPreupdateEqual(
   int iCol;                       /* Used to iterate through columns */
   u8 *a = pChange->aRecord;       /* Cursor used to scan change record */
 
+  if( pTab->bRowid ){
+    if( a[0]!=SQLITE_INTEGER ) return 0;
+    return sessionGetI64(&a[1])==iRowid;
+  }
+
   assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE );
   for(iCol=0; iCol<pTab->nCol; iCol++){
     if( !pTab->abPK[iCol] ){
@@ -997,7 +1013,8 @@ static int sessionTableInfo(
   int *pnCol,                     /* OUT: number of columns */
   const char **pzTab,             /* OUT: Copy of zThis */
   const char ***pazCol,           /* OUT: Array of column names for table */
-  u8 **pabPK                      /* OUT: Array of booleans - true for PK col */
+  u8 **pabPK,                     /* OUT: Array of booleans - true for PK col */
+  int *pbRowid                    /* OUT: True if only PK is a rowid */
 ){
   char *zPragma;
   sqlite3_stmt *pStmt;
@@ -1009,6 +1026,7 @@ static int sessionTableInfo(
   u8 *pAlloc = 0;
   char **azCol = 0;
   u8 *abPK = 0;
+  int bRowid = 0;                 /* Set to true to use rowid as PK */
 
   assert( pazCol && pabPK );
 
@@ -1053,10 +1071,15 @@ static int sessionTableInfo(
   }
 
   nByte = nThis + 1;
+  bRowid = (pbRowid!=0);
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
     nByte += sqlite3_column_bytes(pStmt, 1);
     nDbCol++;
+    if( sqlite3_column_int(pStmt, 5) ) bRowid = 0;
   }
+  if( nDbCol==0 ) bRowid = 0;
+  nDbCol += bRowid;
+  nByte += strlen(SESSIONS_ROWID);
   rc = sqlite3_reset(pStmt);
 
   if( rc==SQLITE_OK ){
@@ -1078,6 +1101,14 @@ static int sessionTableInfo(
     }
   
     i = 0;
+    if( bRowid ){
+      size_t nName = strlen(SESSIONS_ROWID);
+      memcpy(pAlloc, SESSIONS_ROWID, nName+1);
+      azCol[i] = (char*)pAlloc;
+      pAlloc += nName+1;
+      abPK[i] = 1;
+      i++;
+    }
     while( SQLITE_ROW==sqlite3_step(pStmt) ){
       int nName = sqlite3_column_bytes(pStmt, 1);
       const unsigned char *zName = sqlite3_column_text(pStmt, 1);
@@ -1089,7 +1120,6 @@ static int sessionTableInfo(
       i++;
     }
     rc = sqlite3_reset(pStmt);
-  
   }
 
   /* If successful, populate the output variables. Otherwise, zero them and
@@ -1106,6 +1136,7 @@ static int sessionTableInfo(
     if( pzTab ) *pzTab = 0;
     sessionFree(pSession, azCol);
   }
+  if( pbRowid ) *pbRowid = bRowid;
   sqlite3_finalize(pStmt);
   return rc;
 }
@@ -1127,7 +1158,8 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){
     u8 *abPK;
     assert( pTab->azCol==0 || pTab->abPK==0 );
     pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, 
-        pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK
+        pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK,
+        (pSession->bImplicitPK ? &pTab->bRowid : 0)
     );
     if( pSession->rc==SQLITE_OK ){
       int i;
@@ -1199,6 +1231,7 @@ static int sessionUpdateMaxSize(
 ){
   i64 nNew = 2;
   if( pC->op==SQLITE_INSERT ){
+    if( pTab->bRowid ) nNew += 9;
     if( op!=SQLITE_DELETE ){
       int ii;
       for(ii=0; ii<pTab->nCol; ii++){
@@ -1215,12 +1248,16 @@ static int sessionUpdateMaxSize(
   }else{
     int ii;
     u8 *pCsr = pC->aRecord;
-    for(ii=0; ii<pTab->nCol; ii++){
+    if( pTab->bRowid ){
+      nNew += 9 + 1;
+      pCsr += 9;
+    }
+    for(ii=pTab->bRowid; ii<pTab->nCol; ii++){
       int bChanged = 1;
       int nOld = 0;
       int eType;
       sqlite3_value *p = 0;
-      pSession->hook.xNew(pSession->hook.pCtx, ii, &p);
+      pSession->hook.xNew(pSession->hook.pCtx, ii-pTab->bRowid, &p);
       if( p==0 ){
         return SQLITE_NOMEM;
       }
@@ -1299,6 +1336,7 @@ static int sessionUpdateMaxSize(
 */
 static void sessionPreupdateOneChange(
   int op,                         /* One of SQLITE_UPDATE, INSERT, DELETE */
+  i64 iRowid,
   sqlite3_session *pSession,      /* Session object pTab is attached to */
   SessionTable *pTab              /* Table that change applies to */
 ){
@@ -1314,7 +1352,7 @@ static void sessionPreupdateOneChange(
 
   /* Check the number of columns in this xPreUpdate call matches the 
   ** number of columns in the table.  */
-  if( pTab->nCol!=pSession->hook.xCount(pSession->hook.pCtx) ){
+  if( (pTab->nCol-pTab->bRowid)!=pSession->hook.xCount(pSession->hook.pCtx) ){
     pSession->rc = SQLITE_SCHEMA;
     return;
   }
@@ -1347,14 +1385,16 @@ static void sessionPreupdateOneChange(
   /* Calculate the hash-key for this change. If the primary key of the row
   ** includes a NULL value, exit early. Such changes are ignored by the
   ** session module. */
-  rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull);
+  rc = sessionPreupdateHash(
+      pSession, iRowid, pTab, op==SQLITE_INSERT, &iHash, &bNull
+  );
   if( rc!=SQLITE_OK ) goto error_out;
 
   if( bNull==0 ){
     /* Search the hash table for an existing record for this row. */
     SessionChange *pC;
     for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){
-      if( sessionPreupdateEqual(pSession, pTab, pC, op) ) break;
+      if( sessionPreupdateEqual(pSession, iRowid, pTab, pC, op) ) break;
     }
 
     if( pC==0 ){
@@ -1369,7 +1409,7 @@ static void sessionPreupdateOneChange(
   
       /* Figure out how large an allocation is required */
       nByte = sizeof(SessionChange);
-      for(i=0; i<pTab->nCol; i++){
+      for(i=0; i<(pTab->nCol-pTab->bRowid); i++){
         sqlite3_value *p = 0;
         if( op!=SQLITE_INSERT ){
           TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p);
@@ -1384,6 +1424,9 @@ static void sessionPreupdateOneChange(
         rc = sessionSerializeValue(0, p, &nByte);
         if( rc!=SQLITE_OK ) goto error_out;
       }
+      if( pTab->bRowid ){
+        nByte += 9;               /* Size of rowid field - an integer */
+      }
   
       /* Allocate the change object */
       pC = (SessionChange *)sessionMalloc64(pSession, nByte);
@@ -1400,7 +1443,12 @@ static void sessionPreupdateOneChange(
       ** required values and encodings have already been cached in memory.
       ** It is not possible for an OOM to occur in this block. */
       nByte = 0;
-      for(i=0; i<pTab->nCol; i++){
+      if( pTab->bRowid ){
+        pC->aRecord[0] = SQLITE_INTEGER;
+        sessionPutI64(&pC->aRecord[1], iRowid);
+        nByte = 9;
+      }
+      for(i=0; i<(pTab->nCol-pTab->bRowid); i++){
         sqlite3_value *p = 0;
         if( op!=SQLITE_INSERT ){
           pSession->hook.xOld(pSession->hook.pCtx, i, &p);
@@ -1515,9 +1563,10 @@ static void xPreUpdate(
     pSession->rc = sessionFindTable(pSession, zName, &pTab);
     if( pTab ){
       assert( pSession->rc==SQLITE_OK );
-      sessionPreupdateOneChange(op, pSession, pTab);
+      assert( op==SQLITE_UPDATE || iKey1==iKey2 );
+      sessionPreupdateOneChange(op, iKey1, pSession, pTab);
       if( op==SQLITE_UPDATE ){
-        sessionPreupdateOneChange(SQLITE_INSERT, pSession, pTab);
+        sessionPreupdateOneChange(SQLITE_INSERT, iKey2, pSession, pTab);
       }
     }
   }
@@ -1556,6 +1605,7 @@ static void sessionPreupdateHooks(
 typedef struct SessionDiffCtx SessionDiffCtx;
 struct SessionDiffCtx {
   sqlite3_stmt *pStmt;
+  int bRowid;
   int nOldOff;
 };
 
@@ -1564,17 +1614,17 @@ struct SessionDiffCtx {
 */
 static int sessionDiffOld(void *pCtx, int iVal, sqlite3_value **ppVal){
   SessionDiffCtx *p = (SessionDiffCtx*)pCtx;
-  *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff);
+  *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff+p->bRowid);
   return SQLITE_OK;
 }
 static int sessionDiffNew(void *pCtx, int iVal, sqlite3_value **ppVal){
   SessionDiffCtx *p = (SessionDiffCtx*)pCtx;
-  *ppVal = sqlite3_column_value(p->pStmt, iVal);
+  *ppVal = sqlite3_column_value(p->pStmt, iVal+p->bRowid);
    return SQLITE_OK;
 }
 static int sessionDiffCount(void *pCtx){
   SessionDiffCtx *p = (SessionDiffCtx*)pCtx;
-  return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt);
+  return (p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt)) - p->bRowid;
 }
 static int sessionDiffDepth(void *pCtx){
   (void)pCtx;
@@ -1653,14 +1703,16 @@ static char *sessionExprCompareOther(
 static char *sessionSelectFindNew(
   const char *zDb1,      /* Pick rows in this db only */
   const char *zDb2,      /* But not in this one */
+  int bRowid,
   const char *zTbl,      /* Table name */
   const char *zExpr
 ){
+  const char *zSel = (bRowid ? SESSIONS_ROWID ", *" : "*");
   char *zRet = sqlite3_mprintf(
-      "SELECT * FROM \"%w\".\"%w\" WHERE NOT EXISTS ("
+      "SELECT %s FROM \"%w\".\"%w\" WHERE NOT EXISTS ("
       "  SELECT 1 FROM \"%w\".\"%w\" WHERE %s"
       ")",
-      zDb1, zTbl, zDb2, zTbl, zExpr
+      zSel, zDb1, zTbl, zDb2, zTbl, zExpr
   );
   return zRet;
 }
@@ -1674,7 +1726,9 @@ static int sessionDiffFindNew(
   char *zExpr
 ){
   int rc = SQLITE_OK;
-  char *zStmt = sessionSelectFindNew(zDb1, zDb2, pTab->zName,zExpr);
+  char *zStmt = sessionSelectFindNew(
+      zDb1, zDb2, pTab->bRowid, pTab->zName, zExpr
+  );
 
   if( zStmt==0 ){
     rc = SQLITE_NOMEM;
@@ -1685,8 +1739,10 @@ static int sessionDiffFindNew(
       SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx;
       pDiffCtx->pStmt = pStmt;
       pDiffCtx->nOldOff = 0;
+      pDiffCtx->bRowid = pTab->bRowid;
       while( SQLITE_ROW==sqlite3_step(pStmt) ){
-        sessionPreupdateOneChange(op, pSession, pTab);
+        i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0);
+        sessionPreupdateOneChange(op, iRowid, pSession, pTab);
       }
       rc = sqlite3_finalize(pStmt);
     }
@@ -1696,6 +1752,27 @@ static int sessionDiffFindNew(
   return rc;
 }
 
+/*
+** Return a comma-separated list of the fully-qualified (with both database
+** and table name) column names from table pTab. e.g.
+**
+**    "main"."t1"."a", "main"."t1"."b", "main"."t1"."c"
+*/
+static char *sessionAllCols(
+  const char *zDb,
+  SessionTable *pTab
+){
+  int ii;
+  char *zRet = 0;
+  for(ii=0; ii<pTab->nCol; ii++){
+    zRet = sqlite3_mprintf("%z%s\"%w\".\"%w\".\"%w\"",
+        zRet, (zRet ? ", " : ""), zDb, pTab->zName, pTab->azCol[ii] 
+    );
+    if( !zRet ) break;
+  }
+  return zRet;
+}
+
 static int sessionDiffFindModified(
   sqlite3_session *pSession, 
   SessionTable *pTab, 
@@ -1710,11 +1787,13 @@ static int sessionDiffFindModified(
   if( zExpr2==0 ){
     rc = SQLITE_NOMEM;
   }else{
+    char *z1 = sessionAllCols(pSession->zDb, pTab);
+    char *z2 = sessionAllCols(zFrom, pTab);
     char *zStmt = sqlite3_mprintf(
-        "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)",
-        pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2
+        "SELECT %s,%s FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)",
+        z1, z2, pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2
     );
-    if( zStmt==0 ){
+    if( zStmt==0 || z1==0 || z2==0 ){
       rc = SQLITE_NOMEM;
     }else{
       sqlite3_stmt *pStmt;
@@ -1725,12 +1804,15 @@ static int sessionDiffFindModified(
         pDiffCtx->pStmt = pStmt;
         pDiffCtx->nOldOff = pTab->nCol;
         while( SQLITE_ROW==sqlite3_step(pStmt) ){
-          sessionPreupdateOneChange(SQLITE_UPDATE, pSession, pTab);
+          i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0);
+          sessionPreupdateOneChange(SQLITE_UPDATE, iRowid, pSession, pTab);
         }
         rc = sqlite3_finalize(pStmt);
       }
-      sqlite3_free(zStmt);
     }
+    sqlite3_free(zStmt);
+    sqlite3_free(z1);
+    sqlite3_free(z2);
   }
 
   return rc;
@@ -1769,9 +1851,12 @@ int sqlite3session_diff(
       int bHasPk = 0;
       int bMismatch = 0;
       int nCol;                   /* Columns in zFrom.zTbl */
+      int bRowid = 0;
       u8 *abPK;
       const char **azCol = 0;
-      rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK);
+      rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK, 
+          pSession->bImplicitPK ? &bRowid : 0
+      );
       if( rc==SQLITE_OK ){
         if( pTo->nCol!=nCol ){
           bMismatch = 1;
@@ -2320,7 +2405,7 @@ static int sessionAppendUpdate(
     /* If at least one field has been modified, this is not a no-op. */
     if( bChanged ) bNoop = 0;
 
-    /* Add a field to the old.* record. This is omitted if this modules is
+    /* Add a field to the old.* record. This is omitted if this module is
     ** currently generating a patchset. */
     if( bPatchset==0 ){
       if( bChanged || abPK[i] ){
@@ -2422,6 +2507,7 @@ static int sessionSelectStmt(
   int bIgnoreNoop,
   const char *zDb,                /* Database name */
   const char *zTab,               /* Table name */
+  int bRowid,
   int nCol,                       /* Number of columns in table */
   const char **azCol,             /* Names of table columns */
   u8 *abPK,                       /* PRIMARY KEY  array */
@@ -2430,7 +2516,7 @@ static int sessionSelectStmt(
   int rc = SQLITE_OK;
   char *zSql = 0;
   const char *zSep = "";
-  const char *zCols = "*";
+  const char *zCols = bRowid ? SESSIONS_ROWID ", *" : "*";
   int nSql = -1;
   int i;
 
@@ -2449,7 +2535,6 @@ static int sessionSelectStmt(
     zCols = "tbl, ?2, stat";
   }else{
     for(i=0; i<nCol; i++){
-
       if( abPK[i] ){
         sessionAppendStr(&pkfield, zSep, &rc);
         sessionAppendStr(&pkvar, zSep, &rc);
@@ -2656,10 +2741,18 @@ static int sessionGenerateChangeset(
       sqlite3_stmt *pSel = 0;     /* SELECT statement to query table pTab */
       int nRewind = buf.nBuf;     /* Initial size of write buffer */
       int nNoop;                  /* Size of buffer after writing tbl header */
+      int bRowid = 0;
 
       /* Check the table schema is still Ok. */
-      rc = sessionTableInfo(0, db, pSession->zDb, zName, &nCol, 0,&azCol,&abPK);
-      if( !rc && (pTab->nCol!=nCol || memcmp(abPK, pTab->abPK, nCol)) ){
+      rc = sessionTableInfo(
+          0, db, pSession->zDb, zName, &nCol, 0, &azCol, &abPK, 
+          (pSession->bImplicitPK ? &bRowid : 0)
+      );
+      if( rc==SQLITE_OK && (
+          pTab->nCol!=nCol 
+       || pTab->bRowid!=bRowid 
+       || memcmp(abPK, pTab->abPK, nCol)
+      )){
         rc = SQLITE_SCHEMA;
       }
 
@@ -2669,7 +2762,7 @@ static int sessionGenerateChangeset(
       /* Build and compile a statement to execute: */
       if( rc==SQLITE_OK ){
         rc = sessionSelectStmt(
-            db, 0, pSession->zDb, zName, nCol, azCol, abPK, &pSel
+            db, 0, pSession->zDb, zName, bRowid, nCol, azCol, abPK, &pSel
         );
       }
 
@@ -2753,7 +2846,7 @@ int sqlite3session_changeset(
   int rc;
 
   if( pnChangeset==0 || ppChangeset==0 ) return SQLITE_MISUSE;
-  rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset,ppChangeset);
+  rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset);
   assert( rc || pnChangeset==0 
        || pSession->bEnableSize==0 || *pnChangeset<=pSession->nMaxChangesetSize 
   );
@@ -2871,6 +2964,19 @@ int sqlite3session_object_config(sqlite3_session *pSession, int op, void *pArg){
       break;
     }
 
+    case SQLITE_SESSION_OBJCONFIG_ROWID: {
+      int iArg = *(int*)pArg;
+      if( iArg>=0 ){
+        if( pSession->pTable ){
+          rc = SQLITE_MISUSE;
+        }else{
+          pSession->bImplicitPK = (iArg!=0);
+        }
+      }
+      *(int*)pArg = pSession->bImplicitPK;
+      break;
+    }
+
     default:
       rc = SQLITE_MISUSE;
   }
@@ -3860,6 +3966,7 @@ struct SessionApplyCtx {
   u8 bRebaseStarted;              /* If table header is already in rebase */
   u8 bRebase;                     /* True to collect rebase information */
   u8 bIgnoreNoop;                 /* True to ignore no-op conflicts */
+  int bRowid;
 };
 
 /* Number of prepared UPDATE statements to cache. */
@@ -4110,8 +4217,9 @@ static int sessionSelectRow(
   const char *zTab,               /* Table name */
   SessionApplyCtx *p              /* Session changeset-apply context */
 ){
+  /* TODO */
   return sessionSelectStmt(db, p->bIgnoreNoop,
-      "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect
+      "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect
   );
 }
 
@@ -4807,6 +4915,7 @@ static int sessionChangesetApply(
       sApply.bStat1 = 0;
       sApply.bDeferConstraints = 1;
       sApply.bRebaseStarted = 0;
+      sApply.bRowid = 0;
       memset(&sApply.constraints, 0, sizeof(SessionBuffer));
 
       /* If an xFilter() callback was specified, invoke it now. If the 
@@ -4826,8 +4935,8 @@ static int sessionChangesetApply(
         int i;
 
         sqlite3changeset_pk(pIter, &abPK, 0);
-        rc = sessionTableInfo(0, 
-            db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK
+        rc = sessionTableInfo(0, db, "main", zNew, 
+            &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK, &sApply.bRowid
         );
         if( rc!=SQLITE_OK ) break;
         for(i=0; i<sApply.nCol; i++){

ext/session/sqlite3session.h

@@ -80,16 +80,20 @@ int sqlite3session_create(
 void sqlite3session_delete(sqlite3_session *pSession);
 
 /*
-** CAPIREF: Conigure a Session Object
+** CAPI3REF: Configure a Session Object
 ** METHOD: sqlite3_session
 **
 ** This method is used to configure a session object after it has been
-** created. At present the only valid value for the second parameter is
-** [SQLITE_SESSION_OBJCONFIG_SIZE].
+** created. At present the only valid values for the second parameter are
+** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID].
 **
-** Arguments for sqlite3session_object_config()
+*/
+int sqlite3session_object_config(sqlite3_session*, int op, void *pArg);
+
+/*
+** CAPI3REF: Options for sqlite3session_object_config
 **
-** The following values may passed as the the 4th parameter to
+** The following values may passed as the the 2nd parameter to
 ** sqlite3session_object_config().
 **
 ** <dt>SQLITE_SESSION_OBJCONFIG_SIZE <dd>
@@ -105,12 +109,21 @@ void sqlite3session_delete(sqlite3_session *pSession);
 **
 **   It is an error (SQLITE_MISUSE) to attempt to modify this setting after 
 **   the first table has been attached to the session object.
+**
+** <dt>SQLITE_SESSION_OBJCONFIG_ROWID <dd>
+**   This option is used to set, clear or query the flag that enables
+**   collection of data for tables with no explicit PRIMARY KEY.
+**
+**   Normally, tables with no explicit PRIMARY KEY are simply ignored
+**   by the sessions module. However, if this flag is set, it behaves
+**   as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted
+**   as their leftmost columns.
+**
+**   It is an error (SQLITE_MISUSE) to attempt to modify this setting after 
+**   the first table has been attached to the session object.
 */
-int sqlite3session_object_config(sqlite3_session*, int op, void *pArg);
-
-/*
-*/
-#define SQLITE_SESSION_OBJCONFIG_SIZE 1
+#define SQLITE_SESSION_OBJCONFIG_SIZE  1
+#define SQLITE_SESSION_OBJCONFIG_ROWID 2
 
 /*
 ** CAPI3REF: Enable Or Disable A Session Object

ext/session/test_session.c

@@ -76,9 +76,11 @@ int sql_exec_changeset(
 ){
   sqlite3_session *pSession = 0;
   int rc;
+  int val = 1;
 
   /* Create a new session object */
   rc = sqlite3session_create(db, "main", &pSession);
+  sqlite3session_object_config(pSession, SQLITE_SESSION_OBJCONFIG_ROWID, &val);
 
   /* Configure the session object to record changes to all tables */
   if( rc==SQLITE_OK ) rc = sqlite3session_attach(pSession, NULL);
@@ -260,7 +262,7 @@ static int SQLITE_TCLAPI test_session_cmd(
     { "diff",         2, "FROMDB TBL", }, /* 8 */
     { "memory_used",  0, "",           }, /* 9 */
     { "changeset_size", 0, "",         }, /* 10 */
-    { "object_config_size", 1, "INTEGER", }, /* 11 */
+    { "object_config", 2, "OPTION INTEGER", }, /* 11 */
     { 0 }
   };
   int iSub;
@@ -379,15 +381,27 @@ static int SQLITE_TCLAPI test_session_cmd(
       Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize));
       break;
     }
-    case 11: {
+    case 11: {    /* object_config */
+      struct ObjConfOpt {
+        const char *zName;
+        int opt;
+      } aOpt[] = {
+        { "size", SQLITE_SESSION_OBJCONFIG_SIZE },
+        { "rowid", SQLITE_SESSION_OBJCONFIG_ROWID },
+        { 0, 0 }
+      };
+      size_t sz = sizeof(aOpt[0]);
+
       int rc;
       int iArg;
-      if( Tcl_GetIntFromObj(interp, objv[2], &iArg) ){
+      int iOpt;
+      if( Tcl_GetIndexFromObjStruct(interp,objv[2],aOpt,sz,"option",0,&iOpt) ){
         return TCL_ERROR;
       }
-      rc = sqlite3session_object_config(
-          pSession, SQLITE_SESSION_OBJCONFIG_SIZE, &iArg
-      );
+      if( Tcl_GetIntFromObj(interp, objv[3], &iArg) ){
+        return TCL_ERROR;
+      }
+      rc = sqlite3session_object_config(pSession, aOpt[iOpt].opt, &iArg);
       if( rc!=SQLITE_OK ){
         extern const char *sqlite3ErrName(int);
         Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));

ext/wasm/api/sqlite3-api-prologue.js

@@ -1537,6 +1537,9 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
      Full docs: https://sqlite.org/c3ref/db_config.html
 
      Returns capi.SQLITE_MISUSE if op is not a valid operation ID.
+
+     The variants which take `(int, int*)` arguments treat a
+     missing or falsy pointer argument as 0.
   */
   capi.sqlite3_db_config = function(pDb, op, ...args){
     if(!this.s){
@@ -1565,6 +1568,8 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
         case capi.SQLITE_DBCONFIG_ENABLE_VIEW:
         case capi.SQLITE_DBCONFIG_LEGACY_FILE_FORMAT:
         case capi.SQLITE_DBCONFIG_TRUSTED_SCHEMA:
+        case capi.SQLITE_DBCONFIG_STMT_SCANSTATUS:
+        case capi.SQLITE_DBCONFIG_REVERSE_SCANORDER:
           return this.ip(pDb, op, args[0], args[1] || 0);
         case capi.SQLITE_DBCONFIG_LOOKASIDE:
           return this.pii(pDb, op, args[0], args[1], args[2]);

ext/wasm/api/sqlite3-vfs-opfs.c-pp.js

@@ -353,7 +353,6 @@ const installOpfsVfs = function callee(options){
       state.opIds.xClose = i++;
       state.opIds.xDelete = i++;
       state.opIds.xDeleteNoWait = i++;
-      state.opIds.xFileControl = i++;
       state.opIds.xFileSize = i++;
       state.opIds.xLock = i++;
       state.opIds.xOpen = i++;
@@ -718,12 +717,9 @@ const installOpfsVfs = function callee(options){
         return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
       },
       xFileControl: function(pFile, opId, pArg){
-        mTimeStart('xFileControl');
-        const rc = (capi.SQLITE_FCNTL_SYNC===opId)
-              ? opRun('xSync', pFile, 0)
-              : capi.SQLITE_NOTFOUND;
-        mTimeEnd();
-        return rc;
+        /*mTimeStart('xFileControl');
+          mTimeEnd();*/
+        return capi.SQLITE_NOTFOUND;
       },
       xFileSize: function(pFile,pSz64){
         mTimeStart('xFileSize');
@@ -779,8 +775,11 @@ const installOpfsVfs = function callee(options){
         return rc;
       },
       xSync: function(pFile,flags){
+        mTimeStart('xSync');
         ++metrics.xSync.count;
-        return 0; // impl'd in xFileControl()
+        const rc = opRun('xSync', pFile, flags);
+        mTimeEnd();
+        return rc;
       },
       xTruncate: function(pFile,sz64){
         mTimeStart('xTruncate');
@@ -1189,7 +1188,15 @@ const installOpfsVfs = function callee(options){
             /* Truncate journal mode is faster than delete for
                this vfs, per speedtest1. That gap seems to have closed with
                Chrome version 108 or 109, but "persist" is very roughly 5-6%
-               faster than truncate in initial tests. */
+               faster than truncate in initial tests.
+
+               For later analysis: Roy Hashimoto notes that TRUNCATE
+               and PERSIST modes may decrease OPFS concurrency because
+               multiple connections can open the journal file in those
+               modes:
+
+               https://github.com/rhashimoto/wa-sqlite/issues/68
+            */
             "pragma journal_mode=persist;",
             /*
               This vfs benefits hugely from cache on moderate/large
@@ -1279,14 +1286,15 @@ const installOpfsVfs = function callee(options){
             promiseReject(new Error(data.payload.join(' ')));
             break;
           case 'opfs-async-loaded':
-            /*Arrives as soon as the asyc proxy finishes loading.
-              Pass our config and shared state on to the async worker.*/
+            /* Arrives as soon as the asyc proxy finishes loading.
+               Pass our config and shared state on to the async
+               worker. */
             W.postMessage({type: 'opfs-async-init',args: state});
             break;
-          case 'opfs-async-inited':{
-            /*Indicates that the async partner has received the 'init'
-              and has finished initializing, so the real work can
-              begin...*/
+          case 'opfs-async-inited': {
+            /* Indicates that the async partner has received the 'init'
+               and has finished initializing, so the real work can
+               begin... */
             if(true===promiseWasRejected){
               break /* promise was already rejected via timer */;
             }
@@ -1321,10 +1329,15 @@ const installOpfsVfs = function callee(options){
             }
             break;
           }
-          default:
-            promiseReject(e);
-            error("Unexpected message from the async worker:",data);
+          default: {
+            const errMsg = (
+              "Unexpected message from the OPFS async worker: " +
+              JSON.stringify(data)
+            );
+            error(errMsg);
+            promiseReject(new Error(errMsg));
             break;
+          }
       }/*switch(data.type)*/
     }/*W.onmessage()*/;
   })/*thePromise*/;

ext/wasm/api/sqlite3-wasm.c

@@ -567,6 +567,8 @@ const char * sqlite3_wasm_enum_json(void){
     DefInt(SQLITE_DBCONFIG_ENABLE_VIEW);
     DefInt(SQLITE_DBCONFIG_LEGACY_FILE_FORMAT);
     DefInt(SQLITE_DBCONFIG_TRUSTED_SCHEMA);
+    DefInt(SQLITE_DBCONFIG_STMT_SCANSTATUS);
+    DefInt(SQLITE_DBCONFIG_REVERSE_SCANORDER);
     DefInt(SQLITE_DBCONFIG_MAX);
   } _DefGroup;
 
@@ -1545,6 +1547,8 @@ int sqlite3_wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){
     case SQLITE_DBCONFIG_ENABLE_VIEW:
     case SQLITE_DBCONFIG_LEGACY_FILE_FORMAT:
     case SQLITE_DBCONFIG_TRUSTED_SCHEMA:
+    case SQLITE_DBCONFIG_STMT_SCANSTATUS:
+    case SQLITE_DBCONFIG_REVERSE_SCANORDER:
       return sqlite3_db_config(pDb, op, arg1, pArg2);
     default: return SQLITE_MISUSE;
   }

ext/wasm/speedtest1-worker.html

@@ -23,7 +23,7 @@
     </figure>
     <div class="emscripten" id="module-status">Downloading...</div>
     <div class="emscripten">
-      <progress value="0" max="100" id="module-progress" hidden='1'></progress>  
+      <progress value="0" max="100" id="module-progress" hidden='1'></progress>
     </div><!-- /emscripten bits -->
     <fieldset id='ui-controls' class='hidden'>
       <legend>Options</legend>
@@ -68,10 +68,9 @@
       <ul>
         <li>Control-click the flags to (de)select multiple flags.</li>
         <li>The <tt>--big-transactions</tt> flag is important for two
-          of the bigger tests. Without it, those tests create a
-          combined total of 140k implicit transactions, reducing their
-          speed to an absolute crawl, especially when WASMFS is
-          activated.
+          of the bigger tests. Without it, those tests create many
+          thousands of implicit transactions, reducing the affected
+          tests to an absolute crawl, in particular with OPFS.
         </li>
         <li>The easiest way to try different optimization levels is,
           from this directory:

manifest

@@ -1,5 +1,5 @@
-C Try\sto\suse\sa\sheap\sto\smake\scoalescence\sof\sadjacent\sfree\sslots\sfaster\swhen\nfreeing\sspace\son\sa\sbtree\spage.\s\sTurns\sout\sthat\sthe\soverhead\sof\smanaging\nthe\sheap\soverwhelms\sany\sperformance\sgain\sand\sthe\sresult\sis\sslower.
-D 2023-04-07T11:55:58.282
+C Add\ssupport\sfor\sthe\scomma\s(,)\smodifier\sto\s%f\sformats\sin\sthe\sformat()\sfunction.
+D 2023-05-04T13:07:49.375
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,14 +86,14 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h c0d46e399e345e35985b72a1c1af025973bfaa5b1e3563b0ce3bb0ce144a7ca3
+F ext/fts5/fts5Int.h ed48a096418ff4a7c02ac9bd1e8d40c46de21b79a132b8b08d3f32233703de7d
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 501e7d3566bc92766b0e11c0109a7c5a6146bc41144195459af5422f6c2078aa
-F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989
+F ext/fts5/fts5_config.c 051056a9052f5d3a4d1c695f996fd364f920e341f136c60ab2c04aa7e267113f
+F ext/fts5/fts5_expr.c 7d298d76ea010c339b26ca47f6f69e9aef520ea46c083deaa4e83e87cf0e94b1
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c df5b29576a409f673e54b470723d817df9d5167cff208c48ab9a3773cba6fa89
-F ext/fts5/fts5_main.c fe67b6fb2ef134d9dbfa3941c63f777d755b075449be1863cb913a7f8754cb69
+F ext/fts5/fts5_index.c de3cdae2e0056594aad97a728be5c43b6d7a6cdc7e9cc16f197892b2d8689c21
+F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d
 F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -104,7 +104,7 @@ F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d0988
 F ext/fts5/fts5_vocab.c 12138e84616b56218532e3e8feb1d3e0e7ae845e33408dbe911df520424dc9d6
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
+F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b
 F ext/fts5/test/fts5aa.test 5bd43427b7d08ce2e19c488a26534be450538b9232d4d5305049e8de236e9aa9
 F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18ca9385880bb7
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
@@ -137,6 +137,7 @@ F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36
 F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3
 F ext/fts5/test/fts5corrupt5.test 550d0884c14424f9acad051a741f1dd99ec9342277d938e91ff3daf9123d1209
 F ext/fts5/test/fts5corrupt6.test bf8eeae07825b088b9665d9d8e4accbd8dc9bf3cb85b6c64cf6c9e18ccc420a4
+F ext/fts5/test/fts5corrupt7.test f3e68673af2514e31dd67a2ed163f7f597252ab683dec155b8db0cdc0b668342
 F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7
 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4
 F ext/fts5/test/fts5determin.test 1b77879b2ae818b5b71c859e534ee334dac088b7cf3ff3bf76a2c82b1c788d11
@@ -168,7 +169,7 @@ F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc27826807405
 F ext/fts5/test/fts5matchinfo.test 10c9a6f7fe61fb132299c4183c012770b10c4d5c2f2edb6df0b6607f683d737a
 F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6
 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2
-F ext/fts5/test/fts5misc.test d6d4fdd7ec164e69e50af539137c0565362a4124547bf841ba474f092298637b
+F ext/fts5/test/fts5misc.test c02f3e78aa7e62891b5f711c498a877f03252f10a6974e23bc722533233f2603
 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581
 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45
 F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd
@@ -187,6 +188,13 @@ F ext/fts5/test/fts5rebuild.test 55d6f17715cddbf825680dd6551efbc72ed916d8cf1cde4
 F ext/fts5/test/fts5restart.test 835ecc8f449e3919f72509ab58056d0cedca40d1fe04108ccf8ac4c2ba41f415
 F ext/fts5/test/fts5rowid.test b8790ec170a8dc1942a15aef3db926a5f3061b1ff171013003d8297203a20ad6
 F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e332d654375690f8e6e43f
+F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64
+F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc
+F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610
+F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97
+F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
+F ext/fts5/test/fts5secure6.test 7a959d834be6725c641b3c3b38ef86570ea671216ad803e054e4fdff33a72ce2
+F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b
 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
 F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f
@@ -204,7 +212,7 @@ F ext/fts5/test/fts5unicode3.test 590c72e18195bda2446133f9d82d04a4e89d094bba58c7
 F ext/fts5/test/fts5unicode4.test 6463301d669f963c83988017aa354108be0b947d325aef58d3abddf27147b687
 F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db63892db6bafabbec21af4d
 F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d9f005b5a80ccf0bc
-F ext/fts5/test/fts5version.test c8f2cc105f0abf0224965f93e584633dee3e06c91478bc67e468f7cfdf97fd6a
+F ext/fts5/test/fts5version.test d6e5a5897550afeccc2f8531d87404dc1c289ee89385dd4318dbdd75e71d7a67
 F ext/fts5/test/fts5vocab.test 7ed80d9af1ddaaa1637da05e406327b5aac250848bc604c1c1cc667908b87760
 F ext/fts5/test/fts5vocab2.test 681980e92e031c9f3fe8d9c149189e876c108da2fb0fb3a25bd8a9b94bff8f68
 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85
@@ -264,8 +272,8 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23
 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
 F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824
-F ext/misc/base64.c e83a915fcb94c9332e9a92aa4c3beafe2552bd3da2813fc5fff31918cca0b834
-F ext/misc/base85.c 77dfd5813d23ea561d0348f922583888e78f8eaeb2b9a4a28226d092389890b8
+F ext/misc/base64.c a71b131e50300c654a66c469a25b62874481f3d1cb3beb56aca9a68edd812e0d
+F ext/misc/base85.c 073054111988db593ef5fdb87ab8c459df1ea0c3aaaddf0f5bfa3d72b7e6280a
 F ext/misc/basexx.c 5e859e1820620aa8080fb9145eb47089de426ae808f6abb01a8e12921c3a8e67
 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a
 F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9
@@ -294,13 +302,14 @@ F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d
 F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691
 F ext/misc/prefixes.c 0f4f8cff5aebc00a7e3ac4021fd59cfe1a8e17c800ceaf592859ecb9cbc38196
 F ext/misc/qpvtab.c 09738419e25f603a35c0ac8bd0a04daab794f48d08a9bc07a6085b9057b99009
+F ext/misc/randomjson.c 7dd13664155319d47b9facc0d8dbf45e13062966a47168e54e3f26d48240d7ea
 F ext/misc/regexp.c f50ab59bfa8934b7ed98de069d2c74c187f2ef523fb09e85f8840f6459a90942
 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
 F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c
 F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946
-F ext/misc/series.c 8d79354f2c3d46b95ee21272a07cf0bcabb58d1f2b06d9e7b8a31dca1dacb3e5
+F ext/misc/series.c 37d27377684d3ea14177540d2f2767163197611eaba905790c96abd4ab552cd3
 F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d
-F ext/misc/shathree.c 9b7af7b7d55b27c5bbd16548b67fe6aa47a819e71bc6a80a456144f86f4e834d
+F ext/misc/shathree.c 543af7ce71d391cd3a9ab6924a6a1124efc63211fd0f2e240dc4b56077ba88ac
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 94df9bbfa514a563c1484f684a2df3d128a2f7209a84ca3ca100c68a0163e29f
 F ext/misc/sqlar.c 53e7d48f68d699a24f1a92e68e71eca8b3a9ff991fe9588c2a05bde103c6e7b7
@@ -316,7 +325,7 @@ F ext/misc/vfsstat.c 474d08efc697b8eba300082cb1eb74a5f0f3df31ed257db1cb07e72ab0e
 F ext/misc/vtablog.c 5538acd0c8ddaae372331bee11608d76973436b77d6a91e8635cfc9432fba5ae
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c a838d1bea913c514ff316c69695efbb49ea3b8cb37d22afc57f73b6b010b4546
-F ext/misc/zipfile.c b9d615e1d9af7577833861cfaa79b253aec0f26c89239c75af8c790d287d1d39
+F ext/misc/zipfile.c b1f36004c19fb5f949fb166fc4ab88e96a86f66629e9ddb4736a45b63fc3d553
 F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64
 F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8
 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255
@@ -341,7 +350,7 @@ F ext/rbu/rbucrash.test d2b5d619d9281c89cad74401b73b46172daa89906940b1d739c813dd
 F ext/rbu/rbucrash2.test 0a1a72223d880215ce2893a3260320c31a9358d23cb124c610e4f0d984a93285
 F ext/rbu/rbudiff.test 8b8b8b569c68fc880134e0fac4bf6b4b7a907aea4cc6eacf7e1d45e1d47b6aac
 F ext/rbu/rbudor.test 293a192e668bb8e9c7c9704b080c1086ee17496f768e0f1823049e7d02651d1b
-F ext/rbu/rbuexlock.test e3ece733cc8e0a6cb08533a41ed6f562438539ed309028c63375a5adee4a263b
+F ext/rbu/rbuexlock.test e07a0875d0b72f7c007e5d5dcf424e9d48a4752a1a9bcee8ee36947e6add6d5b
 F ext/rbu/rbuexpr.test 2c91617509c88b6e9030f7bf6ff720df26032fcd801adc25533feae726a57382
 F ext/rbu/rbufault.test c51de14067cfe867849530d3d1718ffeb28522f28d52937f95dd7bc2116eb42e
 F ext/rbu/rbufault2.test 8cc8f6298d2d7d20080b2c77e65b607af8b89839f9d87c0972b27e6442edc258
@@ -362,13 +371,13 @@ F ext/rbu/rbuvacuum.test 542561741ff2b262e3694bc6012b44694ee62c545845319a06f3237
 F ext/rbu/rbuvacuum2.test ae097d04feb041446a74fac94b24bffeb3fdd60e32b848c5611e507ab702b81b
 F ext/rbu/rbuvacuum3.test 3ce42695fdf21aaa3499e857d7d4253bc499ad759bcd6c9362042c13cd37d8de
 F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69eefaebb205
-F ext/rbu/sqlite3rbu.c 348bb6251e6ec459de102f8b2dd50789a98643ef7a28e56e4c787ac9659c15ea
+F ext/rbu/sqlite3rbu.c 71a7f0dea3a846ff7c2499dc34a2528f5ddcbe23e2c54dc3cd1fa4d933377c6d
 F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304
 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055
 F ext/recover/dbdata.c 31d580785cf14eb3c20ed6fbb421a10a66569858f837928e6b326088c38d4c72
-F ext/recover/recover1.test 2072993624d5e32fef20ae03b17fc06c02bcb344421fe17bb329b24d2a51e647
+F ext/recover/recover1.test 36a4fd33134809439e677c0e7045b1d31a0ae4f70f464830bce158e2fb1e82f8
 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a
-F ext/recover/recoverbuild.test a6f05273ff5fe517afd166444597c70cb97033e7f58496433a4428a1ecb5d79f
+F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3
 F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e
 F ext/recover/recovercorrupt.test 64c081ad1200ae77b447da99eb724785d6bf71715f394543dc7689642e92bf49
 F ext/recover/recovercorrupt2.test 74bef7dd2d7dd4856f3da21be6e213d27da44827e0f5f0946ca0325b46d163ed
@@ -379,7 +388,7 @@ F ext/recover/recoverpgsz.test 3658ab8e68475b1bb87d6af88baa04551c84b73280a566a1b
 F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f
 F ext/recover/recoverslowidx.test 5205a9742dd9490ee99950dabb622307355ef1662dea6a3a21030057bfd81411
 F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486
-F ext/recover/sqlite3recover.c 3346e73e8570cb803618f0d4b65a0d07a3e246c6591347bdf04c7d0bb6d3bbde
+F ext/recover/sqlite3recover.c b3813090c97df34858bab8ece4e8c49d9e2f56ba2b9bb019f57f44bc234b3c6d
 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959
 F ext/recover/test_recover.c 1a34e2d04533d919a30ae4d5caeb1643f6684e9ccd7597ca27721d8af81f4ade
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
@@ -393,7 +402,7 @@ F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c3350
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/geopoly.c 971e0b5bd9adaf0811feb8c0842a310811159da10319eb0e74fdb42bf26b99ca
-F ext/rtree/rtree.c e05929e78d127613a9eea5dc372c77a049484892d8e9fac1fe0cce85ce4fba81
+F ext/rtree/rtree.c 925888f7672b326fc2a29b3a212ec3ae4aa2331507ecccfaf7f0ac0335020330
 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412
 F ext/rtree/rtree1.test d47f58832145fcfed9067bc457ca8664962196c4566c17a1ebd679367db55d11
 F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d
@@ -442,12 +451,12 @@ F ext/session/session9.test 5409d90d8141881d08285ed1c2c0d8d10fb92069
 F ext/session/sessionA.test 1feeab0b8e03527f08f2f1defb442da25480138f
 F ext/session/sessionB.test c4fb7f8a688787111606e123a555f18ee04f65bb9f2a4bb2aa71d55ce4e6d02c
 F ext/session/sessionC.test f8a5508bc059ae646e5ec9bdbca66ad24bc92fe99fda5790ac57e1f59fce2fdf
-F ext/session/sessionD.test 4f91d0ca8afc4c3969c72c9f0b5ea9527e21de29039937d0d973f821e8470724
+F ext/session/sessionD.test f5c6a762d00bc6ca9d561695c322ba8ecca2bed370486707ef37cf565d2f6c73
 F ext/session/sessionE.test b2010949c9d7415306f64e3c2072ddabc4b8250c98478d3c0c4d064bce83111d
 F ext/session/sessionF.test d37ed800881e742c208df443537bf29aa49fd56eac520d0f0c6df3e6320f3401
 F ext/session/sessionG.test 3efe388282d641b65485b5462e67851002cd91a282dc95b685d085eb8efdad0a
 F ext/session/sessionH.test 71bbff6b1abb2c4ac62b84dee53273c37e0b21e5fde3aed80929403e091ef859
-F ext/session/session_common.tcl db0dda567c75950604072251744e9a6ad5795a3009963c44eb8510f23a8cda64
+F ext/session/session_common.tcl e5598096425486b363718e2cda48ee85d660c96b4f8ea9d9d7a4c3ef514769da
 F ext/session/session_speed_test.c dcf0ef58d76b70c8fbd9eab3be77cf9deb8bc1638fed8be518b62d6cbdef88b3
 F ext/session/sessionat.test 00c8badb35e43a2f12a716d2734a44d614ff62361979b6b85419035bc04b45ee
 F ext/session/sessionbig.test 47c381e7acfabeef17d98519a3080d69151723354d220afa2053852182ca7adf
@@ -458,13 +467,14 @@ F ext/session/sessioninvert.test 04075517a9497a80d39c495ba6b44f3982c7371129b89e2
 F ext/session/sessionmem.test f2a735db84a3e9e19f571033b725b0b2daf847f3f28b1da55a0c1a4e74f1de09
 F ext/session/sessionnoop.test a9366a36a95ef85f8a3687856ebef46983df399541174cb1ede2ee53b8011bc7
 F ext/session/sessionnoop2.test 5c9a882219e54711c98dccd2fd81392f189a59325e4fb5d8ed25e33a0c2f0ba2
-F ext/session/sessionrebase.test ccfa716b23bd1d3b03217ee58cfd90c78d4b99f53e6a9a2f05e82363b9142810
-F ext/session/sessionsize.test 6f644aff31c7f1e4871e9ff3542766e18da68fc7e587b83a347ea9820a002dd8
+F ext/session/sessionrebase.test 702378bdcb5062f1106e74457beca8797d09c113a81768734a58b197b5b334e2
+F ext/session/sessionrowid.test 6323ba831721205fd729929745038fd54e9d128c66c654b8d0b26853095a321c
+F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795
 F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c
 F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc
-F ext/session/sqlite3session.c 1795263b72c1a17e48e95a131a69543af3fa31aa8e81271c7c5cb0911f063604
-F ext/session/sqlite3session.h c367c3043dbb57f69cca35258ebbeadb24e8738980b1a1ae1e281c1b0fac3989
-F ext/session/test_session.c b55a669a2150eb7c491b8b42c69a3eed9bc895cf5fea371a2c813b9618f72163
+F ext/session/sqlite3session.c e50a9218ee360db0a25298adc6614162d80ebe65d3f6a5b0a021e0902f6536a1
+F ext/session/sqlite3session.h 653e9d49c4edae231df8a4c8d69c2145195aedb32462d4b44229dbee7d2680fb
+F ext/session/test_session.c 5285482f83cd92b4c1fe12fcf88210566a18312f4f2aa110f6399dae46aeccbb
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
@@ -484,14 +494,14 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af
 F ext/wasm/api/sqlite3-api-cleanup.js cc21e3486da748463e02bbe51e2464c6ac136587cdfd5aa00cd0b5385f6ca808
 F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803
 F ext/wasm/api/sqlite3-api-oo1.js 2691a34a741015127b210954a1b9586764d3ff0c8a20f00fd15c00f339ecc79f
-F ext/wasm/api/sqlite3-api-prologue.js 461ffa5a95f4c1935b3970a58790d3cdca62b16e9b9a6a8d993a2a47d7561f51
+F ext/wasm/api/sqlite3-api-prologue.js 17f4ec398ba34c5c666fea8e8c4eb82064a35b302f2f2eb355283cd8d3f68ed5
 F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d660194ddfc51eab1c5b9bf
 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
 F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c42cc345903c81b5fae661ce512f
 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487
-F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 5cb63f847737ded723d0bdf9b3313e8f7deb0bbfd0082dedde540991edde155b
+F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 89640e4874a60cb2d973306b272384ffb45c7915375c7bb0355c7586f88dc39c
 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
-F ext/wasm/api/sqlite3-wasm.c c42413ca9f3e64c424b2bbfc5decf639670ca38bc8f7afb7760d5379398c9307
+F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda
 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc
 F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75
 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8
@@ -525,7 +535,7 @@ F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234
 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06
 F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18
 F ext/wasm/speedtest1-wasmfs.html 7a301f4f5b6ad4f5d37fd6e7ca03a2f5d5547fd289da60a39075a93d7646d354
-F ext/wasm/speedtest1-worker.html fe6b36a63de1012bb9fb4d2fb888b6de9c589c21b0aa3ae054459b0093e077bf
+F ext/wasm/speedtest1-worker.html 82869822e641c1bef3ec0cd2d7d2b6a42d0b4f68a7b160fb2e1dd0b523940a9b
 F ext/wasm/speedtest1-worker.js 13b57c4a41729678a1194014afec2bd5b94435dcfc8d1039dfa9a533ac819ee1
 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
@@ -557,16 +567,16 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2
-F src/alter.c 3ca2f449c890f8b86ec9e06f0c4fccf0648941c3308a16904cb2852227db83f7
-F src/analyze.c 01bfd40026632eaae1d93212b684f539c6674cb573535dc90199674cbf7e0cdc
+F src/alter.c 482c534877fbb543f8295992cde925df55443febac5db5438d5aaba6f78c4940
+F src/analyze.c a1f3061af16c99f73aed0362160176c31a6452de1b02ada1d68f6839f2a37df0
 F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39
 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf
-F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7
+F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523
 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d
 F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca
-F src/btree.c 8bd2f2b5e41d9ebb6e9f828f597e824af41c3a48315543027d741fbbc1fa89b0
+F src/btree.c 1949007d2792cf761799348b3760c8489d1964c49efa9e52591aa1f2fcdbc96f
 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc
-F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8
+F src/btreeInt.h a9ae91868acc4b3146d47ae2a072aac2cf41ecb7386015752160c8e1a212d9f2
 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54
 F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
@@ -575,20 +585,20 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7
 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387
 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef
 F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873
-F src/expr.c 874702ffa80dec44cd4b5dd456627136750e1f585568d51febbdb1f4a7f21970
+F src/expr.c 871cfd80c516ee39d90414b2d3da2b5bc9c9e21fe87b7eb787ea7ae4b6461758
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
-F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002
+F src/fkey.c 03c134cc8bffe54835f742ddea0b72ebfc8f6b32773d175c71b8afeea6cb5c83
 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122
-F src/global.c 428d2580a1cdf5dbe1f356d1feab83710ae0cc862ece0fb57bc8259e43838c74
+F src/global.c bd0892ade7289f6e20bff44c07d06371f2ff9b53cea359e7854b9b72f65adc30
 F src/hash.c c6af5f96a7a76d000f07c5402c48c318c2566beecdee9e78b9d9f60ce7119565
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4
-F src/json.c edae65fe1f66ce8b1e7fa6eb036a3d8cf525dacd91d58f12068e80a81ac34f61
+F src/json.c 7297dbd1d623850578c21bb8a99b87e745d09e14fd36ebc965ace67c86f902b4
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136
-F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d
+F src/main.c 035be2e9ba2a0fc1701a8ab1880af3001a968a24556433538a6c073558ee4341
 F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -597,7 +607,7 @@ F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6
 F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff
 F src/memdb.c 559c42e61eb70cd6d4bc692b042497133c6d96c09a3d514d92f3dac72268e223
 F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0
-F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8
+F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9
 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25
 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a
 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
@@ -609,33 +619,33 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06
 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a
 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107
-F src/os_unix.c 90c4fa0a88c8b0817c7ce4dbea0ac3aebfd7deb0797945ac34dfd25006ba393a
-F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345
+F src/os_unix.c 1b3ddb7814c4bf37f494c04d2ab30c1ced5b2c927267e1930ce7cd388787a96d
+F src/os_win.c 2b2411279f7b24f927591561303fc5871845732df42641cbf695c23640b16975
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 39af8ff7c73a991f61f4d1e3a6f8f98b1c8e29144723507822774cac5e6ee0b5
+F src/pager.c 9d36ddedc842e993c88c222ed914822dbd6f8ece3c648fde04468637012a034a
 F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3
-F src/parse.y 424e49ed8fc6c907920db9be5a13a75ed43811e1ea8dd21b0fa9ef97f083dc6b
-F src/pcache.c 842410539b544e12d5fccfcf29890782f46a58f227a77bc0bd76243799662c0c
+F src/parse.y 146f9a1db7db5ef4299c6897d335e5abed348c2626190d2877d45ffa210fd4ca
+F src/pcache.c 8ee13acccfd9accbf0af94910b7323dd7f7d55300d92ddafcf40e34fcc8e21be
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc
 F src/pragma.c 26ed2cfdc5c12aa1c707178635709684960288cacc9cff9d491a38ff10e395f1
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
-F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3
-F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623
+F src/prepare.c 6350675966bd0e7ac3a464af9dbfe26db6f0d4237f4e1f1acdb17b12ad371e6e
+F src/printf.c 19a25adf1b73892d41af7d8f7cbc55b01b592bf2062e68b9f10e604d8deee7e0
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
-F src/select.c df007d60326a9aad94f5a4915d01e7de19d77627ed09338747a2e8e741a75add
-F src/shell.c.in 2140c98b8185ce5f024d706a552dd0ee861c35621dab6599a9234019348bf9dc
-F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a
+F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27
+F src/shell.c.in 39ea3d9c17c65c42c6c415222d89a32ae683b245c8af7b4bfc544d9246055d16
+F src/sqlite.h.in d6b0b83b2deab8f92ef7cc6f6fb94fa59d21c59f7b55f4c693bfff161ce42238
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
-F src/sqliteInt.h 899781baef0d1dd0910524df6350e0ef7e2761131f6e04ec5e34f3b32e262998
+F src/sqliteInt.h 91303fb4ee858b85ae1a8a48cc8f723339b81ba7138b42ee5c000083bfff0934
 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
 F src/tclsqlite.c 8522a04fb9c84faa1d80354430ae0ee9349727a3a4b32e3cfe39b9be8324cabd
-F src/test1.c 74d87bbc29954c239fdcd7c0c3dd472c0eb9761c7289b85389f322233932780a
+F src/test1.c 8eab61fb2813aa212d97ab188e85fc9ca7b89d9ff5ff05d59d9aa0c491a6c721
 F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef
 F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644
 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664
@@ -675,7 +685,7 @@ F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b
 F src/test_schema.c f5d6067dfc2f2845c4dd56df63e66ee826fb23877855c785f75cc2ca83fd0c1b
 F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a
 F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e
-F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939
+F src/test_syscall.c 9fdb13b1df05e639808d44fcb8f6064aaded32b6565c00b215cfd05a060d1aca
 F src/test_tclsh.c 3ff5d188a72f00807425954ea3b493dfd3a4b890ecc6700ea83bad2fd1332ecf
 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc
 F src/test_thread.c 7ddcf0c8b79fa3c1d172f82f322302c963d923cdb503c6171f3c8081586d0b01
@@ -693,28 +703,28 @@ F src/trigger.c ad6ab9452715fa9a8075442e15196022275b414b9141b566af8cdb7a1605f2b0
 F src/update.c 3f4fb5ad7c9b48d7911974d6579192bb3a6c27f46140b6cbb9139cc8a77b8691
 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
-F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f
+F src/util.c d4bcb560471cd94e6e17d448311f8d5bf81a7e5276295a53501058ef1b95dd1a
 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd
-F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b
-F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c
+F src/vdbe.c 94d5520d2a287216c47e6fb641ee88ffd934b0d40c235d693d38bcd0e0750357
+F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884
 F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb
-F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339
-F src/vdbeaux.c 0379f2529aa16ab82fefddb1163c3138d6552ede80962b599ae711bbf5777608
+F src/vdbeapi.c b4982cde547054c4f7341198db3c3008a48e1eb028f757601bf5bf2fc026cbcf
+F src/vdbeaux.c 6ee48db408d4c297a363f1e31145c09793a580e7c508bb36063dd017d67117a2
 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd
-F src/vdbemem.c db0458d11a51f6cfad2333a41e36a3795be0b2f316d070df5d33543a9ac884ac
+F src/vdbemem.c 1cac4028c0dabbf1f3259f107440e2780e05ac9fe419e9709e6eb4e166ba714b
 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35
 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac
 F src/vtab.c 4a1b231b5938de0282fbb605eb068ca673a1b6a383130f54d1bcbbac951988ad
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d
+F src/wal.c 7a65f64bfe4a783c5e2df73ffb0efc383dec934dee9e3ac706b2eeb3631d17ac
 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
 F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 16ba2073aa2139bf97576c49baa6189a33200e123cbc86374a7410a0aed5a117
+F src/where.c f69d94f34e1c523cd9b66041e4afe015cad29888617f3c09a2a5bc36018917d0
 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c
-F src/wherecode.c 41c17b089082558c1c5496e8a453b1f9a96485b6b355f318440fca2ee4754b2e
-F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63
-F src/window.c 76a27cff9ea2ded0c2c3527187029259440fabcc4cc4c07b11d942c78494a614
+F src/wherecode.c b300db0bcd84ad6c2642bf3f509f92fad7b7d697b9856b64dd66d692d184d054
+F src/whereexpr.c 22cf19b0ececeaf838daed1039c5231a8778784eba5ad67b991442a23473fd3f
+F src/window.c e075ea85bea322e30e361fa6e69eddba74f461e99e2a564dc09973f8a1fb27d9
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627
 F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9
@@ -739,13 +749,13 @@ F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74
 F test/altermalloc2.test 17fb3724c4b004c469c27dc4ef181608aa644555fbd3f3236767584f73747c81
 F test/altermalloc3.test 8531b3086f0a7889f43971a579a8c81832d5123f4703d8c86b89ba1136c63b9a
 F test/alterqf.test ff6c6f881485c29ed699b8ef4774864ca1b0c01a6c08f5cdd624a008e4b40fca
-F test/altertab.test 7273b8506eab46342be016af78028df49f3bd99037412f997a8f1011b37a6912
+F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27
 F test/altertab2.test 62597b6fd08feaba1b6bfe7d31dac6117c67e06dc9ce9c478a3abe75b5926de0
 F test/altertab3.test 6c432fbb9963e0bd6549bf1422f6861d744ee5a80cb3298564e81e556481df16
 F test/altertrig.test fb5951d21a2c954be3b8a8cf8e10b5c0fa20687c53fd67d63cea88d08dd058d5
 F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f
 F test/analyze.test 547bb700f903107b38611b014ca645d6b5bb819f5210d7bf39c40802aafeb7d7
-F test/analyze3.test 4440c4932247adb2b4e0c838f657c19dc7af4f56859255436dc4e855f39b9324
+F test/analyze3.test 03f4b3d794760cf15da2d85a52df9bae300e51c8fefe9c36cfae1f86dc10d23f
 F test/analyze4.test 68bd069f3ac7ac1e652ddd9f04f57d5606ddb4208450f5297005db7aa0dd707d
 F test/analyze5.test fa5131952303ac4146aba101b116b9c8cb89e2637531c334a6df7f7d19dddc0d
 F test/analyze6.test 028f5bdfc9e5b5294768fa9a7185b8cd1d019aa7aab5b2f8ee42d7271d9a3b28
@@ -788,7 +798,7 @@ F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4
 F test/avfs.test 0c3a38e03cccb0fc3127838462dc05dc3f4c1480d770c084b388304c25de3652
 F test/avtrans.test b7dc25459ecbd86c6fa9c606ee3068f59d81e225118617dcf2bbb6ded2ade89e
 F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d
-F test/backup.test fc1ecefce723fad5199b55cec7a5a992ec8c3ad6873419e5e8919066dec457f3
+F test/backup.test 3b08fd4af69f0fa786931103a31f4542b184aba16e239e5f22b18c3c2476697f
 F test/backup2.test 8facb54df1388419d34b362ab1f7e233310ff3a3af64e8ad5ec47ba3c2bbe5cf
 F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32
 F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4
@@ -912,7 +922,7 @@ F test/createtab.test 85cdfdae5c3de331cd888d6c66e1aba575b47c2e3c3cc4a1d6f5414069
 F test/cse.test 00b3aea44b16828833c94fbe92475fd6977583fcb064ae0bc590986812b38d0c
 F test/csv01.test 2ab5514005fd308995c8910bc313e47f0368b94213b9d6c27f9a2da78796a091
 F test/ctime.test 340f362f41f92972bbd71f44e10569a5cc694062b692231bd08aa6fe6c1c4773
-F test/cursorhint.test 12e7d03262be8705798fe7b5719956bcda14989f10296a5e8fdc13466b7b75fc
+F test/cursorhint.test 05cf0febe5c5f8a31f199401fd1c9322249e753950d55f26f9d5aca61408a270
 F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f
 F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8
 F test/date.test 118e04db8c8b4efeb885542b4918c7b869a34c460a6bebbfe927dfd75706b80d
@@ -991,7 +1001,7 @@ F test/filter1.test 590f8ba9a0cd0823b80d89ac75c5ce72276189cef9225d2436adaf1ee87f
 F test/filter2.tcl 44e525497ce07382915f01bd29ffd0fa49dab3adb87253b5e5103ba8f93393e8
 F test/filter2.test 485cf95d1f6d6ceee5632201ca52a71868599836f430cdee42e5f7f14666e30a
 F test/filterfault.test c08fb491d698e8df6c122c98f7db1c65ffcfcad2c1ab0e07fa8a5be1b34eaa8b
-F test/fkey1.test 55663090ab6735319a52647057b9f19f8ec8c6c7d7da25170b71a75e3e5bdeb7
+F test/fkey1.test e563bcb4cb108ce3f40363cda4f84009dc89a39e2973076e5057ba99fca35378
 F test/fkey2.test 1063d65e5923c054cfb8f0555a92a3ae0fa8c067275a33ee1715bd856cdb304c
 F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49
 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d
@@ -1195,7 +1205,7 @@ F test/index7.test b238344318e0b4e42126717f6554f0e7dfd0b39cecad4b736039b43e1e3b6
 F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a912a3
 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997
 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0
-F test/indexexpr1.test b2a15637dcbae7fd8d7e2fc51f74ac4feaf5510130ee2089a5ec5bd1ef7270e1
+F test/indexexpr1.test 62558b1cfd7ccbe7bc015849cc6d1a13ef124e80cbd5b3a98dc66c3c9cce0cf4
 F test/indexexpr2.test 1c382e81ef996d8ae8b834a74f2a9013dddf59214c32201d7c8a656d739f999a
 F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
@@ -1220,7 +1230,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
 F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9
-F test/join.test ed1daf99958fed1b9f017e56bae2bb6b49339a1ec0b70b9e8f7259960c6bf387
+F test/join.test aea7a4f55b2d9eb8ef3434ea78f55b15bd688ab6136a11105c9c52f77424f199
 F test/join2.test 8561fe82ce434ac96de91544072e578dc2cadddf2d9bc9cd802f866a9b92502e
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
@@ -1235,18 +1245,24 @@ F test/joinC.test 1f1a602c2127f55f136e2cbd3bf2d26546614bf8cffe5902ec1ac9c07f87f2
 F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be28
 F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b
 F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127
-F test/joinH.test 15f501b33d848521964afde9865a92aeca79c8c41fa84dc4dc3f865c9ed8c868
+F test/joinH.test 705157cf9b9b7c207caf960812a7d0e4dc1dd45aa5fb2b563f12df59088645f3
 F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497
 F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4
 F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ffa140e
 F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946
 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
-F test/json101.test c7707ee623c57a24845ef260d4388a6fade1eb294e450caadd7f1d9ced19dea7
-F test/json102.test 327e77275f338c028faefa2da5164daf6b142a165e3015ff2a6e4251ddc6a0ac
+F test/json/README.md 506af1f54574b524106acb50d1a341ab5ddfa6d83fe25095007892b07e663e85
+F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f
+F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d
+F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x
+F test/json101.test ff8024cbb8092e723237648cea9bdbd51f31476b5015a4df3a5ecc8a5efda837
+F test/json102.test 13dc9e7b7f359ecb861e02f9bd7019f7342a63d1c354273b0a8f3904050560a8
 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe
-F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5
+F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1
 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d
+F test/json501.test f71710f60fa45b19dc336fbaac9e8362f70f80cf81badefdb845ed3f7c7c2ccc
+F test/json502.test 66d150cc098674b8bf4354526a8dd411b926f43ca892306bcb3b6d3f93fef7be
 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff
 F test/kvtest.c feb4358fb022da8ebd098c45811f2f6507688bb6c43aa72b3e840df19026317b
 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63
@@ -1309,7 +1325,7 @@ F test/minmax.test fe638b55d77d2375531a8f549b338eafcd9adfbd2f72df37ed77d9b26ca0a
 F test/minmax2.test cf9311babb6f0518d04e42fd6a42c619531c4309a9dd790a2c4e9b3bc595e0de
 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354
 F test/minmax4.test 272ca395257f05937dc96441c9dde4bc9fbf116a8d4fa02baeb0d13d50e36c87
-F test/misc1.test 9955e70cab5e284d77e358cfa1f1dd43f5e4bc00a421018581b0fbd62206a6a1
+F test/misc1.test 8d138a4926ab90617c1aa29ce26e7785ae2b83a4d3a195d543b7374e05589dd1
 F test/misc2.test 71e746af479119386ac2ed7ab7d81d99970e75b49ffd3e8efffee100b4b5f350
 F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d
 F test/misc4.test 10cd6addb2fa9093df4751a1b92b50440175dd5468a6ec84d0386e78f087db0e
@@ -1484,8 +1500,8 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21
 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707
 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
-F test/shell1.test 291c5c4b313adbe44d847af78b730a3bbfa1598c450e09ffe5601170bf7f28e7
-F test/shell2.test 35c0c19d3198ee7669a748c1aedcce27a776ee575cc76128f8fcf665b79672f7
+F test/shell1.test 300b77328aaafb9f3e7a53a26e4162fbf92181d92251d259ff105a2275ff998d
+F test/shell2.test 09a202f57e7cd99788537f763e0845796a173fcea06a0d199a08d69446fe1daf
 F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a
 F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f
 F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b
@@ -1525,7 +1541,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c
-F test/speedtest1.c 4a883fe277c289161f84f8c805a38f78bfbd9515eb417071f9bce817c2ca8756
+F test/speedtest1.c 9703465daf778dd90bad7adadc1d2ee205ec90a79437cababab0a4111190f0c8
 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
@@ -1557,7 +1573,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309
 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039
 F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d
 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04
-F test/tabfunc01.test 668e5e39108ce84b2f24e402f9382daecc0417edb9c78c6c8f58170584e80c91
+F test/tabfunc01.test 54f27eacd054aa528a8b6e3331192c484104f30aaee351ad035f2b39a00f87c4
 F test/table.test eb3463b7add9f16a5bb836badf118cf391b809d09fdccd1f79684600d07ec132
 F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4
 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930
@@ -1619,7 +1635,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356
 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed
 F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5
 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223
-F test/tkt-99378177930f87bd.test 28530bf9903dcd7743185ce78b1c02b1f9ba09fe4fa77a70ecbd0af83fe3353c
+F test/tkt-99378177930f87bd.test 9d6cff39b50d062c813ae1cb0ebbd1b7acf81ecc23ae5d5215e5bb05667dc137
 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667
 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6
 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8
@@ -1807,7 +1823,7 @@ F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661
 F test/view.test d4c4281e1679245829db35597817282f60dc513fc39cc5439078f009bd118487
 F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679
 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456
-F test/vt02.c 86253b57d6bc2170dfca33f45fd099b66d0bf874e95ecd7786dcbb134f179469
+F test/vt02.c f4a357c8180d71120ca2b466a8df48d9c40fc50873694840327d9647450485f3
 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07
 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082
 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e
@@ -1902,7 +1918,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2
 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c
 F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976
 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc
-F test/window1.test 67aaef6e01e8d8fb3f89aa440e8c8045dbfd3c6f21628a6d7bad2b786c310d9b
+F test/window1.test 5ba48e9d33231e6ef16f21426bade9ccc52abf65a10587bff90a6c14fe174594
 F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476
 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c
 F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03
@@ -1944,7 +1960,7 @@ F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501d
 F test/zeroblob.test 7b74cefc7b281dfa2b07cd237987fbe94b4a2037a7771e9e83f2d5f608b1d99e
 F test/zeroblobfault.test 861d8191a0d944dfebb3cb4d2c5b4e46a5a119eaec5a63dd996c2389f8063441
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 4178a2de98739e9adac41cb8f0be5553df060e470c846f51fdbed247117728a7
+F test/zipfile.test 416adb0ca9bb54f978fe2e77978b1b964ce5e1c0199f45e381caa771e9f8cfc1
 F test/zipfile2.test 9903388a602a3834189857a985106ff95c3bba6a3969e0134127df991889db5d
 F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a168827edbbc2c
 F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb44
@@ -1978,7 +1994,7 @@ F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a19
 F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8
 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x
 F tool/mkctimec.tcl 38e3db33210a200aae791635125052a643a27aa0619a0debf19aa9c55e1b2dde x
-F tool/mkkeywordhash.c 35bfc41adacc4aa6ef6fca7fd0c63e0ec0534b78daf4d0cfdebe398216bbffc3
+F tool/mkkeywordhash.c 9822bd1f58a70e5f84179df3045bba4791df69180e991bec88662703f2e93761
 F tool/mkmsvcmin.tcl 6ecab9fe22c2c8de4d82d4c46797bda3d2deac8e763885f5a38d0c44a895ab33
 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef
 F tool/mkopcodeh.tcl 769d9e6a8b462323150dc13a8539d6064664b72974f7894befe2491cc73e05cd
@@ -2006,7 +2022,7 @@ F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9cc5e66d4ebbf2d194f39db2527ece92077e86ae627ddd233ee48e16e8142564
 F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809
 F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c2a1
-F tool/showwal.c 4699048f68b6dd7b451011abfff404b8890d5a0b7dab78d2ad50d018116239d5
+F tool/showwal.c 65ecabae3a2dcff4116301d5a8dbb8c4964814da1b2aff6d85c806a88b71fa4e
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/spaceanal.tcl 1b5be34c6223cb1af06da2a10fb77863eb869b1962d055820b0a11cf2336ab45
 F tool/speed-check.sh c24c30cddd0ecb6d1d0775411d22f7619f55fa696a305487645e27b1b8fc05a2
@@ -2052,11 +2068,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 27c59f1ea789f3ff245f23e79ded5cd71c48e3a51ffbb8c220b51101a4e69fd7
-R c1ff538937e36abd2cf16de04be70f7b
-T *branch * deadend
-T *sym-deadend *
-T -sym-btree-freespace-opt *
+P d3370d59cffb7ab982d6c620c93d22aa6a9dc786e1c4af95ca8d45ff0b9b7d6f
+R ba5de07dbdc63490b9fe4eac9a9617b8
+T *branch * comma-format
+T *sym-comma-format *
+T -sym-trunk *
 U drh
-Z 10cff07ea83d09e599ad019612e5aecd
+Z 225e8025e1634bf3383f0a288fad9560
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-5d7e833f25051000d24797bc3875aaaed8f2bdb3b9aa1ce3f4f466a7dcd923b8
+7080e196a1f887640ff51ddc508ec6796ce12874c2944855702753b64a8e5e50

src/alter.c

@@ -1279,6 +1279,19 @@ static int renameEditSql(
   return rc;
 }
 
+/*
+** Set all pEList->a[].fg.eEName fields in the expression-list to val.
+*/
+static void renameSetENames(ExprList *pEList, int val){
+  if( pEList ){
+    int i;
+    for(i=0; i<pEList->nExpr; i++){
+      assert( val==ENAME_NAME || pEList->a[i].fg.eEName==ENAME_NAME );
+      pEList->a[i].fg.eEName = val;
+    }
+  }
+}
+
 /*
 ** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming
 ** it was read from the schema of database zDb. Return SQLITE_OK if 
@@ -1326,7 +1339,17 @@ static int renameResolveTrigger(Parse *pParse){
           pSrc = 0;
           rc = SQLITE_NOMEM;
         }else{
+          /* pStep->pExprList contains an expression-list used for an UPDATE
+          ** statement. So the a[].zEName values are the RHS of the
+          ** "<col> = <expr>" clauses of the UPDATE statement. So, before
+          ** running SelectPrep(), change all the eEName values in
+          ** pStep->pExprList to ENAME_SPAN (from their current value of
+          ** ENAME_NAME). This is to prevent any ids in ON() clauses that are
+          ** part of pSrc from being incorrectly resolved against the
+          ** a[].zEName values as if they were column aliases.  */
+          renameSetENames(pStep->pExprList, ENAME_SPAN);
           sqlite3SelectPrep(pParse, pSel, 0);
+          renameSetENames(pStep->pExprList, ENAME_NAME);
           rc = pParse->nErr ? SQLITE_ERROR : SQLITE_OK;
           assert( pStep->pExprList==0 || pStep->pExprList==pSel->pEList );
           assert( pSrc==pSel->pSrc );

src/analyze.c

@@ -1781,6 +1781,10 @@ static int loadStatTbl(
     pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
     assert( pIdx==0 || pIdx->nSample==0 );
     if( pIdx==0 ) continue;
+    if( pIdx->aSample!=0 ){
+      /* The same index appears in sqlite_stat4 under multiple names */
+      continue;
+    }
     assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 );
     if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
       nIdxCol = pIdx->nKeyCol;
@@ -1788,6 +1792,7 @@ static int loadStatTbl(
       nIdxCol = pIdx->nColumn;
     }
     pIdx->nSampleCol = nIdxCol;
+    pIdx->mxSample = nSample;
     nByte = sizeof(IndexSample) * nSample;
     nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample;
     nByte += nIdxCol * sizeof(tRowcnt);     /* Space for Index.aAvgEq[] */
@@ -1827,6 +1832,11 @@ static int loadStatTbl(
     if( zIndex==0 ) continue;
     pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
     if( pIdx==0 ) continue;
+    if( pIdx->nSample>=pIdx->mxSample ){
+      /* Too many slots used because the same index appears in
+      ** sqlite_stat4 using multiple names */
+      continue;
+    }
     /* This next condition is true if data has already been loaded from 
     ** the sqlite_stat4 table. */
     nCol = pIdx->nSampleCol;
@@ -1870,11 +1880,12 @@ static int loadStat4(sqlite3 *db, const char *zDb){
   const Table *pStat4;
 
   assert( db->lookaside.bDisable );
-  if( (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0
+  if( OptimizationEnabled(db, SQLITE_Stat4)
+   && (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0
    && IsOrdinaryTable(pStat4)
   ){
     rc = loadStatTbl(db,
-      "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", 
+      "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx COLLATE nocase", 
       "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4",
       zDb
     );

src/backup.c

@@ -242,13 +242,7 @@ static int backupOnePage(
   assert( !isFatalError(p->rc) );
   assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) );
   assert( zSrcData );
-
-  /* Catch the case where the destination is an in-memory database and the
-  ** page sizes of the source and destination differ. 
-  */
-  if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){
-    rc = SQLITE_READONLY;
-  }
+  assert( nSrcPgsz==nDestPgsz || sqlite3PagerIsMemdb(pDestPager)==0 );
 
   /* This loop runs once for each destination page spanned by the source 
   ** page. For each iteration, variable iOff is set to the byte offset
@@ -381,7 +375,10 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){
     pgszSrc = sqlite3BtreeGetPageSize(p->pSrc);
     pgszDest = sqlite3BtreeGetPageSize(p->pDest);
     destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest));
-    if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){
+    if( SQLITE_OK==rc 
+     && (destMode==PAGER_JOURNALMODE_WAL || sqlite3PagerIsMemdb(pDestPager))
+     && pgszSrc!=pgszDest 
+    ){
       rc = SQLITE_READONLY;
     }
   

src/btree.c

@@ -946,8 +946,25 @@ int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow){
 */
 void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){
   /* Used only by system that substitute their own storage engine */
+#ifdef SQLITE_DEBUG
+  if( ALWAYS(eHintType==BTREE_HINT_RANGE) ){
+    va_list ap;
+    Expr *pExpr;
+    Walker w;
+    memset(&w, 0, sizeof(w));
+    w.xExprCallback = sqlite3CursorRangeHintExprCheck;
+    va_start(ap, eHintType);
+    pExpr = va_arg(ap, Expr*);
+    w.u.aMem = va_arg(ap, Mem*);
+    va_end(ap);
+    assert( pExpr!=0 );
+    assert( w.u.aMem!=0 );
+    sqlite3WalkExpr(&w, pExpr);
+  }
+#endif /* SQLITE_DEBUG */
 }
-#endif
+#endif /* SQLITE_ENABLE_CURSOR_HINTS */
+
 
 /*
 ** Provide flag hints to the cursor.
@@ -1746,7 +1763,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
 ** allocation is being made in order to insert a new cell, so we will
 ** also end up needing a new cell pointer.
 */
-static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){
+static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){
   const int hdr = pPage->hdrOffset;    /* Local cache of pPage->hdrOffset */
   u8 * const data = pPage->aData;      /* Local cache of pPage->aData */
   int top;                             /* First byte of cell content area */
@@ -1862,7 +1879,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
   assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize );
   assert( sqlite3_mutex_held(pPage->pBt->mutex) );
   assert( iSize>=4 );   /* Minimum cell size is 4 */
-  assert( iStart<=pPage->pBt->usableSize-4 );
+  assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 );
 
   /* The list of freeblocks must be in ascending order.  Find the 
   ** spot on the list where iStart should be inserted.
@@ -1919,6 +1936,11 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
   }
   pTmp = &data[hdr+5];
   x = get2byte(pTmp);
+  if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){
+    /* Overwrite deleted information with zeros when the secure_delete
+    ** option is enabled */
+    memset(&data[iStart], 0, iSize);
+  }
   if( iStart<=x ){
     /* The new freeblock is at the beginning of the cell content area,
     ** so just extend the cell content area rather than create another
@@ -1930,14 +1952,9 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
   }else{
     /* Insert the new freeblock into the freelist */
     put2byte(&data[iPtr], iStart);
+    put2byte(&data[iStart], iFreeBlk);
+    put2byte(&data[iStart+2], iSize);
   }
-  if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){
-    /* Overwrite deleted information with zeros when the secure_delete
-    ** option is enabled */
-    memset(&data[iStart], 0, iSize);
-  }
-  put2byte(&data[iStart], iFreeBlk);
-  put2byte(&data[iStart+2], iSize);
   pPage->nFree += iOrigSize;
   return SQLITE_OK;
 }
@@ -7033,7 +7050,6 @@ static int fillInCell(
   return SQLITE_OK;
 }
 
-
 /*
 ** Remove the i-th cell from pPage.  This routine effects pPage only.
 ** The cell content is not freed or deallocated.  It is assumed that
@@ -7097,6 +7113,14 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){
 ** in pTemp or the original pCell) and also record its index. 
 ** Allocating a new entry in pPage->aCell[] implies that 
 ** pPage->nOverflow is incremented.
+**
+** The insertCellFast() routine below works exactly the same as
+** insertCell() except that it lacks the pTemp and iChild parameters
+** which are assumed zero.  Other than that, the two routines are the
+** same.
+**
+** Fixes or enhancements to this routine should be reflected in
+** insertCellFast()!
 */
 static int insertCell(
   MemPage *pPage,   /* Page into which we are copying */
@@ -7119,14 +7143,103 @@ static int insertCell(
   assert( sqlite3_mutex_held(pPage->pBt->mutex) );
   assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB );
   assert( pPage->nFree>=0 );
+  assert( iChild>0 );
   if( pPage->nOverflow || sz+2>pPage->nFree ){
     if( pTemp ){
       memcpy(pTemp, pCell, sz);
       pCell = pTemp;
     }
-    if( iChild ){
-      put4byte(pCell, iChild);
+    put4byte(pCell, iChild);
+    j = pPage->nOverflow++;
+    /* Comparison against ArraySize-1 since we hold back one extra slot
+    ** as a contingency.  In other words, never need more than 3 overflow
+    ** slots but 4 are allocated, just to be safe. */
+    assert( j < ArraySize(pPage->apOvfl)-1 );
+    pPage->apOvfl[j] = pCell;
+    pPage->aiOvfl[j] = (u16)i;
+
+    /* When multiple overflows occur, they are always sequential and in
+    ** sorted order.  This invariants arise because multiple overflows can
+    ** only occur when inserting divider cells into the parent page during
+    ** balancing, and the dividers are adjacent and sorted.
+    */
+    assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */
+    assert( j==0 || i==pPage->aiOvfl[j-1]+1 );   /* Overflows are sequential */
+  }else{
+    int rc = sqlite3PagerWrite(pPage->pDbPage);
+    if( NEVER(rc!=SQLITE_OK) ){
+      return rc;
     }
+    assert( sqlite3PagerIswriteable(pPage->pDbPage) );
+    data = pPage->aData;
+    assert( &data[pPage->cellOffset]==pPage->aCellIdx );
+    rc = allocateSpace(pPage, sz, &idx);
+    if( rc ){ return rc; }
+    /* The allocateSpace() routine guarantees the following properties
+    ** if it returns successfully */
+    assert( idx >= 0 );
+    assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB );
+    assert( idx+sz <= (int)pPage->pBt->usableSize );
+    pPage->nFree -= (u16)(2 + sz);
+    /* In a corrupt database where an entry in the cell index section of
+    ** a btree page has a value of 3 or less, the pCell value might point
+    ** as many as 4 bytes in front of the start of the aData buffer for
+    ** the source page.  Make sure this does not cause problems by not
+    ** reading the first 4 bytes */
+    memcpy(&data[idx+4], pCell+4, sz-4);
+    put4byte(&data[idx], iChild);
+    pIns = pPage->aCellIdx + i*2;
+    memmove(pIns+2, pIns, 2*(pPage->nCell - i));
+    put2byte(pIns, idx);
+    pPage->nCell++;
+    /* increment the cell count */
+    if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++;
+    assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB );
+#ifndef SQLITE_OMIT_AUTOVACUUM
+    if( pPage->pBt->autoVacuum ){
+      int rc2 = SQLITE_OK;
+      /* The cell may contain a pointer to an overflow page. If so, write
+      ** the entry for the overflow page into the pointer map.
+      */
+      ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2);
+      if( rc2 ) return rc2;
+    }
+#endif
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This variant of insertCell() assumes that the pTemp and iChild
+** parameters are both zero.  Use this variant in sqlite3BtreeInsert()
+** for performance improvement, and also so that this variant is only
+** called from that one place, and is thus inlined, and thus runs must
+** faster.
+**
+** Fixes or enhancements to this routine should be reflected into
+** the insertCell() routine.
+*/
+static int insertCellFast(
+  MemPage *pPage,   /* Page into which we are copying */
+  int i,            /* New cell becomes the i-th cell of the page */
+  u8 *pCell,        /* Content of the new cell */
+  int sz            /* Bytes of content in pCell */
+){
+  int idx = 0;      /* Where to write new cell content in data[] */
+  int j;            /* Loop counter */
+  u8 *data;         /* The content of the whole page */
+  u8 *pIns;         /* The point in pPage->aCellIdx[] where no cell inserted */
+
+  assert( i>=0 && i<=pPage->nCell+pPage->nOverflow );
+  assert( MX_CELL(pPage->pBt)<=10921 );
+  assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB );
+  assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) );
+  assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) );
+  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
+  assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB );
+  assert( pPage->nFree>=0 );
+  assert( pPage->nOverflow==0 );
+  if( sz+2>pPage->nFree ){
     j = pPage->nOverflow++;
     /* Comparison against ArraySize-1 since we hold back one extra slot
     ** as a contingency.  In other words, never need more than 3 overflow
@@ -7158,17 +7271,7 @@ static int insertCell(
     assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB );
     assert( idx+sz <= (int)pPage->pBt->usableSize );
     pPage->nFree -= (u16)(2 + sz);
-    if( iChild ){
-      /* In a corrupt database where an entry in the cell index section of
-      ** a btree page has a value of 3 or less, the pCell value might point
-      ** as many as 4 bytes in front of the start of the aData buffer for
-      ** the source page.  Make sure this does not cause problems by not
-      ** reading the first 4 bytes */
-      memcpy(&data[idx+4], pCell+4, sz-4);
-      put4byte(&data[idx], iChild);
-    }else{
-      memcpy(&data[idx], pCell, sz);
-    }
+    memcpy(&data[idx], pCell, sz);
     pIns = pPage->aCellIdx + i*2;
     memmove(pIns+2, pIns, 2*(pPage->nCell - i));
     put2byte(pIns, idx);
@@ -7478,38 +7581,6 @@ static int pageInsertArray(
   return 0;
 }
 
-static void btreeHeapInsert(u32 *aHeap, u32 x);
-static int btreeHeapPull(u32 *aHeap, u32 *pOut);
-
-/*
-** Add all of the cells on aHep to the freelist for a page.
-*/
-static void btreeFreeCellsOnHeap(MemPage *pPg, u32 *aHeap){
-  u32 x;
-  int iOfst;
-  int iSize;
-
-  assert( aHeap[0] );
-  btreeHeapPull(aHeap, &x);
-  iSize = x & 0xffff;
-  iOfst = x>>16;
-  while( aHeap[0] ){
-    int iNxOfst;
-    int iNxSize;
-    btreeHeapPull(aHeap, &x);
-    iNxSize = x & 0xffff;
-    iNxOfst = x>>16;
-    if( iSize+iOfst==iNxOfst ){
-      iSize += iNxSize;
-    }else{
-      freeSpace(pPg, iOfst, iSize);
-      iOfst = iNxOfst;
-      iSize = iNxSize;
-    }
-  }
-  freeSpace(pPg, iOfst, iSize);
-}
-
 /*
 ** The pCArray object contains pointers to b-tree cells and their sizes.
 **
@@ -7529,28 +7600,51 @@ static int pageFreeArray(
   u8 * const pEnd = &aData[pPg->pBt->usableSize];
   u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize];
   int nRet = 0;
-  int i;
+  int i, j;
   int iEnd = iFirst + nCell;
-  u32 aHeap[100];
+  int nFree = 0;
+  int aOfst[10];
+  int aAfter[10];
 
-  aHeap[0] = 0;
   for(i=iFirst; i<iEnd; i++){
     u8 *pCell = pCArray->apCell[i];
     if( SQLITE_WITHIN(pCell, pStart, pEnd) ){
-      u16 sz;
-      u16 iOfst;
-
+      int sz;
+      int iAfter;
+      int iOfst;
       /* No need to use cachedCellSize() here.  The sizes of all cells that
       ** are to be freed have already been computing while deciding which
       ** cells need freeing */
       sz = pCArray->szCell[i];  assert( sz>0 );
       iOfst = (u16)(pCell - aData);
-      btreeHeapInsert(aHeap, (iOfst<<16)|sz);
-      if( aHeap[0]>=90 ) btreeFreeCellsOnHeap(pPg, aHeap);
+      iAfter = iOfst+sz;
+      for(j=0; j<nFree; j++){
+        if( aOfst[j]==iAfter ){
+          aOfst[j] = iOfst;
+          break;
+        }else if( aAfter[j]==iOfst ){
+          aAfter[j] = iAfter;
+          break;
+        }
+      }
+      if( j>=nFree ){
+        if( nFree>=(int)(sizeof(aOfst)/sizeof(aOfst[0])) ){
+          for(j=0; j<nFree; j++){
+            freeSpace(pPg, aOfst[j], aAfter[j]-aOfst[j]);
+          }
+          nFree = 0;
+        }
+        aOfst[nFree] = iOfst;
+        aAfter[nFree] = iAfter;
+        if( &aData[iAfter]>pEnd ) return 0;
+        nFree++;
+      }
       nRet++;
     }
   }
-  if( aHeap[0] ) btreeFreeCellsOnHeap(pPg, aHeap);
+  for(j=0; j<nFree; j++){
+    freeSpace(pPg, aOfst[j], aAfter[j]-aOfst[j]);
+  }
   return nRet;
 }
 
@@ -9239,7 +9333,7 @@ int sqlite3BtreeInsert(
     }
   }
   assert( pCur->eState==CURSOR_VALID 
-       || (pCur->eState==CURSOR_INVALID && loc) );
+       || (pCur->eState==CURSOR_INVALID && loc) || CORRUPT_DB );
 
   pPage = pCur->pPage;
   assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) );
@@ -9330,7 +9424,7 @@ int sqlite3BtreeInsert(
   }else{
     assert( pPage->leaf );
   }
-  rc = insertCell(pPage, idx, newCell, szNew, 0, 0);
+  rc = insertCellFast(pPage, idx, newCell, szNew);
   assert( pPage->nOverflow==0 || rc==SQLITE_OK );
   assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 );
 

src/btreeInt.h

@@ -172,7 +172,7 @@
 ** byte are used.  The integer consists of all bytes that have bit 8 set and
 ** the first byte with bit 8 clear.  The most significant byte of the integer
 ** appears first.  A variable-length integer may not be more than 9 bytes long.
-** As a special case, all 8 bytes of the 9th byte are used as data.  This
+** As a special case, all 8 bits of the 9th byte are used as data.  This
 ** allows a 64-bit integer to be encoded in 9 bytes.
 **
 **    0x00                      becomes  0x00000000

src/expr.c

@@ -1113,9 +1113,9 @@ Select *sqlite3ExprListToValues(Parse *pParse, int nElem, ExprList *pEList){
 ** Join two expressions using an AND operator.  If either expression is
 ** NULL, then just return the other expression.
 **
-** If one side or the other of the AND is known to be false, then instead
-** of returning an AND expression, just return a constant expression with
-** a value of false.
+** If one side or the other of the AND is known to be false, and neither side
+** is part of an ON clause, then instead of returning an AND expression,
+** just return a constant expression with a value of false.
 */
 Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){
   sqlite3 *db = pParse->db;
@@ -1123,14 +1123,17 @@ Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){
     return pRight;
   }else if( pRight==0 ){
     return pLeft;
-  }else if( (ExprAlwaysFalse(pLeft) || ExprAlwaysFalse(pRight)) 
-         && !IN_RENAME_OBJECT
-  ){
-    sqlite3ExprDeferredDelete(pParse, pLeft);
-    sqlite3ExprDeferredDelete(pParse, pRight);
-    return sqlite3Expr(db, TK_INTEGER, "0");
   }else{
-    return sqlite3PExpr(pParse, TK_AND, pLeft, pRight);
+    u32 f = pLeft->flags | pRight->flags;
+    if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse 
+     && !IN_RENAME_OBJECT
+    ){
+      sqlite3ExprDeferredDelete(pParse, pLeft);
+      sqlite3ExprDeferredDelete(pParse, pRight);
+      return sqlite3Expr(db, TK_INTEGER, "0");
+    }else{
+      return sqlite3PExpr(pParse, TK_AND, pLeft, pRight);
+    }
   }
 }
 
@@ -6288,7 +6291,7 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){
     sqlite3 *db = pParse->db;
     assert( iAgg>=0 );
     if( pExpr->op!=TK_AGG_FUNCTION ){
-      if( ALWAYS(iAgg<pAggInfo->nColumn)
+      if( iAgg<pAggInfo->nColumn
        && pAggInfo->aCol[iAgg].pCExpr==pExpr
       ){
         pExpr = sqlite3ExprDup(db, pExpr, 0);
@@ -6441,7 +6444,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
     default: {
       IndexedExpr *pIEpr;
       Expr tmp;
-      int i;
       assert( pParse->iSelfTab==0 );
       if( (pNC->ncFlags & NC_InAggFunc)==0 ) break;
       if( pParse->pIdxEpr==0 ) break;
@@ -6494,7 +6496,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
           } /* endif pExpr->iTable==pItem->iCursor */
         } /* end loop over pSrcList */
       }
-      return WRC_Prune;
+      return WRC_Continue;
     }
     case TK_AGG_FUNCTION: {
       if( (pNC->ncFlags & NC_InAggFunc)==0

src/fkey.c

@@ -1317,22 +1317,22 @@ static Trigger *fkActionTrigger(
 
     if( action==OE_Restrict ){
       int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
-      Token tFrom;
-      Token tDb;
+      SrcList *pSrc;
       Expr *pRaise; 
 
-      tFrom.z = zFrom;
-      tFrom.n = nFrom;
-      tDb.z = db->aDb[iDb].zDbSName;
-      tDb.n = sqlite3Strlen30(tDb.z);
-
       pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed");
       if( pRaise ){
         pRaise->affExpr = OE_Abort;
       }
+      pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0);
+      if( pSrc ){
+        assert( pSrc->nSrc==1 );
+        pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom);
+        pSrc->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName);
+      }
       pSelect = sqlite3SelectNew(pParse, 
           sqlite3ExprListAppend(pParse, 0, pRaise),
-          sqlite3SrcListAppend(pParse, 0, &tDb, &tFrom),
+          pSrc,
           pWhere,
           0, 0, 0, 0, 0
       );

src/global.c

@@ -97,7 +97,7 @@ const unsigned char *sqlite3aGTb = &sqlite3UpperToLower[256+12-OP_Ne];
 **   isalnum()                        0x06
 **   isxdigit()                       0x08
 **   toupper()                        0x20
-**   SQLite identifier character      0x40
+**   SQLite identifier character      0x40   $, _, or non-ascii
 **   Quote character                  0x80
 **
 ** Bit 0x20 is set if the mapped character requires translation to upper

src/main.c

@@ -3933,7 +3933,7 @@ int sqlite3_sleep(int ms){
   /* This function works in milliseconds, but the underlying OsSleep() 
   ** API uses microseconds. Hence the 1000's.
   */
-  rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000);
+  rc = (sqlite3OsSleep(pVfs, ms<0 ? 0 : 1000*ms)/1000);
   return rc;
 }
 

src/msvc.h

@@ -38,4 +38,8 @@
 #define SQLITE_4_BYTE_ALIGNED_MALLOC
 #endif /* defined(_MSC_VER) && !defined(_WIN64) */
 
+#if !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800
+#define HAVE_LOG2 0
+#endif /* !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 */
+
 #endif /* SQLITE_MSVC_H */

src/os_unix.c

@@ -72,7 +72,7 @@
 #endif
 
 /* Use pread() and pwrite() if they are available */
-#if defined(__APPLE__)
+#if defined(__APPLE__) || defined(__linux__)
 # define HAVE_PREAD 1
 # define HAVE_PWRITE 1
 #endif
@@ -3322,12 +3322,6 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){
 ** Seek to the offset passed as the second argument, then read cnt 
 ** bytes into pBuf. Return the number of bytes actually read.
 **
-** NB:  If you define USE_PREAD or USE_PREAD64, then it might also
-** be necessary to define _XOPEN_SOURCE to be 500.  This varies from
-** one system to another.  Since SQLite does not define USE_PREAD
-** in any form by default, we will not attempt to define _XOPEN_SOURCE.
-** See tickets #2741 and #2681.
-**
 ** To avoid stomping the errno value on a failed read the lastErrno value
 ** is set before returning.
 */

src/os_win.c

@@ -5197,7 +5197,7 @@ static int winOpen(
       if( isReadWrite ){
         int rc2, isRO = 0;
         sqlite3BeginBenignMalloc();
-        rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO);
         sqlite3EndBenignMalloc();
         if( rc2==SQLITE_OK && isRO ) break;
       }
@@ -5214,7 +5214,7 @@ static int winOpen(
       if( isReadWrite ){
         int rc2, isRO = 0;
         sqlite3BeginBenignMalloc();
-        rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO);
         sqlite3EndBenignMalloc();
         if( rc2==SQLITE_OK && isRO ) break;
       }
@@ -5234,7 +5234,7 @@ static int winOpen(
       if( isReadWrite ){
         int rc2, isRO = 0;
         sqlite3BeginBenignMalloc();
-        rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO);
         sqlite3EndBenignMalloc();
         if( rc2==SQLITE_OK && isRO ) break;
       }
@@ -5457,6 +5457,13 @@ static int winAccess(
   OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n",
            zFilename, flags, pResOut));
 
+  if( zFilename==0 ){
+    *pResOut = 0;
+    OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n",
+             zFilename, pResOut, *pResOut));
+    return SQLITE_OK;
+  }
+
   zConverted = winConvertFromUtf8Filename(zFilename);
   if( zConverted==0 ){
     OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename));

src/pager.c

@@ -2605,6 +2605,8 @@ static int pager_truncate(Pager *pPager, Pgno nPage){
   int rc = SQLITE_OK;
   assert( pPager->eState!=PAGER_ERROR );
   assert( pPager->eState!=PAGER_READER );
+  PAGERTRACE(("Truncate %d npage %u\n", PAGERID(pPager), nPage));
+
   
   if( isOpen(pPager->fd) 
    && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) 
@@ -5522,6 +5524,10 @@ static int getPageNormal(
     if( !isOpen(pPager->fd) || pPager->dbSize<pgno || noContent ){
       if( pgno>pPager->mxPgno ){
         rc = SQLITE_FULL;
+        if( pgno<=pPager->dbSize ){
+          sqlite3PcacheRelease(pPg);
+          pPg = 0;
+        }
         goto pager_acquire_err;
       }
       if( noContent ){
@@ -5686,10 +5692,12 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){
 /*
 ** Release a page reference.
 **
-** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be
-** used if we know that the page being released is not the last page.
+** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be used
+** if we know that the page being released is not the last reference to page1.
 ** The btree layer always holds page1 open until the end, so these first
-** to routines can be used to release any page other than BtShared.pPage1.
+** two routines can be used to release any page other than BtShared.pPage1.
+** The assert() at tag-20230419-2 proves that this constraint is always
+** honored.
 **
 ** Use sqlite3PagerUnrefPageOne() to release page1.  This latter routine
 ** checks the total number of outstanding pages and if the number of
@@ -5705,7 +5713,7 @@ void sqlite3PagerUnrefNotNull(DbPage *pPg){
     sqlite3PcacheRelease(pPg);
   }
   /* Do not use this routine to release the last reference to page1 */
-  assert( sqlite3PcacheRefCount(pPager->pPCache)>0 );
+  assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); /* tag-20230419-2 */
 }
 void sqlite3PagerUnref(DbPage *pPg){
   if( pPg ) sqlite3PagerUnrefNotNull(pPg);

src/parse.y

@@ -715,7 +715,7 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N
   seltablist(A) ::= stl_prefix(A) LP seltablist(F) RP as(Z) on_using(N). {
     if( A==0 && Z.n==0 && N.pOn==0 && N.pUsing==0 ){
       A = F;
-    }else if( F->nSrc==1 ){
+    }else if( ALWAYS(F!=0) && F->nSrc==1 ){
       A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,0,&N);
       if( A ){
         SrcItem *pNew = &A->a[A->nSrc-1];
@@ -1378,7 +1378,7 @@ case_else(A) ::=  ELSE expr(X).         {A = X;}
 case_else(A) ::=  .                     {A = 0;} 
 %type case_operand {Expr*}
 %destructor case_operand {sqlite3ExprDelete(pParse->db, $$);}
-case_operand(A) ::= expr(X).            {A = X; /*A-overwrites-X*/} 
+case_operand(A) ::= expr(A).
 case_operand(A) ::= .                   {A = 0;} 
 
 %type exprlist {ExprList*}

src/pcache.c

@@ -70,11 +70,15 @@ struct PCache {
     PgHdr *pPg;
     unsigned char *a;
     int j;
-    pPg = (PgHdr*)pLower->pExtra;
-    printf("%3lld: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags);
-    a = (unsigned char *)pLower->pBuf;
-    for(j=0; j<12; j++) printf("%02x", a[j]);
-    printf(" ptr %p\n", pPg);
+    if( pLower==0 ){
+      printf("%3d: NULL\n", i);
+    }else{
+      pPg = (PgHdr*)pLower->pExtra;
+      printf("%3d: nRef %2lld flgs %02x data ", i, pPg->nRef, pPg->flags);
+      a = (unsigned char *)pLower->pBuf;
+      for(j=0; j<12; j++) printf("%02x", a[j]);
+      printf(" ptr %p\n", pPg);
+    }
   }
   static void pcacheDump(PCache *pCache){
     int N;
@@ -87,9 +91,8 @@ struct PCache {
     if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump;
     for(i=1; i<=N; i++){
        pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0);
-       if( pLower==0 ) continue;
        pcachePageTrace(i, pLower);
-       if( ((PgHdr*)pLower)->pPage==0 ){
+       if( pLower && ((PgHdr*)pLower)->pPage==0 ){
          sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0);
        }
     }

src/prepare.c

@@ -306,7 +306,9 @@ int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFlags){
 #else
       encoding = SQLITE_UTF8;
 #endif
-      if( db->nVdbeActive>0 && encoding!=ENC(db) ){
+      if( db->nVdbeActive>0 && encoding!=ENC(db)
+       && (db->mDbFlags & DBFLAG_Vacuum)==0
+      ){
         rc = SQLITE_LOCKED;
         goto initone_error_out;
       }else{
@@ -700,7 +702,11 @@ static int sqlite3Prepare(
   sParse.db = db;
   sParse.pReprepare = pReprepare;
   assert( ppStmt && *ppStmt==0 );
-  if( db->mallocFailed ) sqlite3ErrorMsg(&sParse, "out of memory");
+  if( db->mallocFailed ){
+    sqlite3ErrorMsg(&sParse, "out of memory");
+    db->errCode = rc = SQLITE_NOMEM;
+    goto end_prepare;
+  }
   assert( sqlite3_mutex_held(db->mutex) );
 
   /* For a long-term use prepared statement avoid the use of

src/printf.c

@@ -649,6 +649,7 @@ void sqlite3_str_vappendf(
         {
           i64 szBufNeeded;           /* Size of a temporary buffer needed */
           szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15;
+          if( cThousand ) szBufNeeded += (e2+2)/3;
           if( szBufNeeded > etBUFSIZE ){
             bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded);
             if( bufpt==0 ) return;
@@ -666,10 +667,12 @@ void sqlite3_str_vappendf(
         }else if( msd>0 ){
           for(; e2>=0; e2--){
             *(bufpt++) = et_getdigit_int(&longvalue,&msd);
+            if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ',';
           }
         }else{
           for(; e2>=0; e2--){
             *(bufpt++) = et_getdigit(&realvalue,&nsd);
+            if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ',';
           }
         }
         /* The decimal point */

src/select.c

@@ -2318,7 +2318,7 @@ void sqlite3SubqueryColumnTypes(
   assert( (pSelect->selFlags & SF_Resolved)!=0 );
   assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 );
   assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB );
-  if( db->mallocFailed ) return;
+  if( db->mallocFailed || IN_RENAME_OBJECT ) return;
   while( pSelect->pPrior ) pSelect = pSelect->pPrior;
   a = pSelect->pEList->a;
   memset(&sNC, 0, sizeof(sNC));
@@ -2363,18 +2363,16 @@ void sqlite3SubqueryColumnTypes(
             break;
           }
         }
-       }
-     }
-     if( zType ){
-       i64 m = sqlite3Strlen30(zType);
-       n = sqlite3Strlen30(pCol->zCnName);
-       pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2);
-       if( pCol->zCnName ){
-         memcpy(&pCol->zCnName[n+1], zType, m+1);
-         pCol->colFlags |= COLFLAG_HASTYPE;
-       }else{
-         testcase( pCol->colFlags & COLFLAG_HASTYPE );
-        pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL);
+      }
+    }
+    if( zType ){
+      i64 m = sqlite3Strlen30(zType);
+      n = sqlite3Strlen30(pCol->zCnName);
+      pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2);
+      pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL);
+      if( pCol->zCnName ){
+        memcpy(&pCol->zCnName[n+1], zType, m+1);
+        pCol->colFlags |= COLFLAG_HASTYPE;
       }
     }
     pColl = sqlite3ExprCollSeq(pParse, p);
@@ -5266,11 +5264,13 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){
       ** use UNION, INTERSECT, or EXCEPT.  Only UNION ALL is allowed. */
       return 0;
     }
+#ifndef SQLITE_OMIT_WINDOWFUNC
     if( pX->pWin ){
       /* This optimization does not work for subqueries that use window
       ** functions. */
       return 0;
     }
+#endif
   }
   colUsed = pItem->colUsed;
   if( pSub->pOrderBy ){
@@ -6446,12 +6446,13 @@ static void optimizeAggregateUseOfIndexedExpr(
   assert( pSelect->pGroupBy!=0 );
   pAggInfo->nColumn = pAggInfo->nAccumulator;
   if( ALWAYS(pAggInfo->nSortingColumn>0) ){
-    if( pAggInfo->nColumn==0 ){
-      pAggInfo->nSortingColumn = pSelect->pGroupBy->nExpr;
-    }else{
-      pAggInfo->nSortingColumn =
-        pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1;
+    int mx = pSelect->pGroupBy->nExpr - 1;
+    int j, k;
+    for(j=0; j<pAggInfo->nColumn; j++){
+      k = pAggInfo->aCol[j].iSorterColumn;
+      if( k>mx ) mx = k;
     }
+    pAggInfo->nSortingColumn = mx+1;
   }
   analyzeAggFuncArgs(pAggInfo, pNC);
 #if TREETRACE_ENABLED

src/shell.c.in

@@ -228,6 +228,7 @@ typedef unsigned char u8;
 #if SQLITE_OS_WINRT
 #include <intrin.h>
 #endif
+#define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 
 /* string conversion routines only needed on Win32 */
@@ -442,11 +443,25 @@ static void endTimer(void){
 static int bail_on_error = 0;
 
 /*
-** Threat stdin as an interactive input if the following variable
+** Treat stdin as an interactive input if the following variable
 ** is true.  Otherwise, assume stdin is connected to a file or pipe.
 */
 static int stdin_is_interactive = 1;
 
+#if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE \
+  && !defined(SHELL_OMIT_WIN_UTF8)
+# define SHELL_WIN_UTF8_OPT 1
+#else
+# define SHELL_WIN_UTF8_OPT 0
+#endif
+
+#if SHELL_WIN_UTF8_OPT
+/*
+** Setup console for UTF-8 input/output when following variable true.
+*/
+static int console_utf8 = 0;
+#endif
+
 /*
 ** On Windows systems we have to know if standard output is a console
 ** in order to translate UTF-8 into MBCS.  The following variable is
@@ -579,16 +594,146 @@ static char *dynamicContinuePrompt(void){
 }
 #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */
 
+#if SHELL_WIN_UTF8_OPT
+/* Following struct is used for -utf8 operation. */
+static struct ConsoleState {
+  int stdinEof;      /* EOF has been seen on console input */
+  int infsMode;      /* Input file stream mode upon shell start */
+  UINT inCodePage;   /* Input code page upon shell start */
+  UINT outCodePage;  /* Output code page upon shell start */
+  HANDLE hConsoleIn; /* Console input handle */
+  DWORD consoleMode; /* Console mode upon shell start */
+} conState = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 };
+
+/*
+** Prepare console, (if known to be a WIN32 console), for UTF-8
+** input (from either typing or suitable paste operations) and for
+** UTF-8 rendering. This may "fail" with a message to stderr, where
+** the preparation is not done and common "code page" issues occur.
+*/
+static void console_prepare(void){
+  HANDLE hCI = GetStdHandle(STD_INPUT_HANDLE);
+  DWORD consoleMode = 0;
+  if( isatty(0) && GetFileType(hCI)==FILE_TYPE_CHAR
+      && GetConsoleMode( hCI, &consoleMode) ){
+    if( !IsValidCodePage(CP_UTF8) ){
+      fprintf(stderr, "Cannot use UTF-8 code page.\n");
+      console_utf8 = 0;
+      return;
+    }
+    conState.hConsoleIn = hCI;
+    conState.consoleMode = consoleMode;
+    conState.inCodePage = GetConsoleCP();
+    conState.outCodePage = GetConsoleOutputCP();
+    SetConsoleCP(CP_UTF8);
+    SetConsoleOutputCP(CP_UTF8);
+    consoleMode |= ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT;
+    SetConsoleMode(conState.hConsoleIn, consoleMode);
+    conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT);
+    console_utf8 = 1;
+  }else{
+    console_utf8 = 0;
+  }
+}
+
+/*
+** Undo the effects of console_prepare(), if any.
+*/
+static void SQLITE_CDECL console_restore(void){
+  if( console_utf8 && conState.inCodePage!=0
+      && conState.hConsoleIn!=INVALID_HANDLE_VALUE ){
+    _setmode(_fileno(stdin), conState.infsMode);
+    SetConsoleCP(conState.inCodePage);
+    SetConsoleOutputCP(conState.outCodePage);
+    SetConsoleMode(conState.hConsoleIn, conState.consoleMode);
+    /* Avoid multiple calls. */
+    conState.hConsoleIn = INVALID_HANDLE_VALUE;
+    conState.consoleMode = 0;
+    console_utf8 = 0;
+  }
+}
+
+/*
+** Collect input like fgets(...) with special provisions for input
+** from the Windows console to get around its strange coding issues.
+** Defers to plain fgets() when input is not interactive or when the
+** startup option, -utf8, has not been provided or taken effect.
+*/
+static char* utf8_fgets(char *buf, int ncmax, FILE *fin){
+  if( fin==0 ) fin = stdin;
+  if( fin==stdin && stdin_is_interactive && console_utf8 ){
+# define SQLITE_IALIM 150
+    wchar_t wbuf[SQLITE_IALIM];
+    int lend = 0;
+    int noc = 0;
+    if( ncmax==0 || conState.stdinEof ) return 0;
+    buf[0] = 0;
+    while( noc<ncmax-7-1 && !lend ){
+      /* There is room for at least 2 more characters and a 0-terminator. */
+      int na = (ncmax > SQLITE_IALIM*4+1 + noc)
+        ? SQLITE_IALIM : (ncmax-1 - noc)/4;
+# undef SQLITE_IALIM
+      DWORD nbr = 0;
+      BOOL bRC = ReadConsoleW(conState.hConsoleIn, wbuf, na, &nbr, 0);
+      if( !bRC || (noc==0 && nbr==0) ) return 0;
+      if( nbr > 0 ){
+        int nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR,
+                                      wbuf,nbr,0,0,0,0);
+        if( nmb !=0 && noc+nmb <= ncmax ){
+          int iseg = noc;
+          nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR,
+                                    wbuf,nbr,buf+noc,nmb,0,0);
+          noc += nmb;
+          /* Fixup line-ends as coded by Windows for CR (or "Enter".)*/
+          if( noc > 0 ){
+            if( buf[noc-1]=='\n' ){
+              lend = 1;
+              if( noc > 1 && buf[noc-2]=='\r' ){
+                buf[noc-2] = '\n';
+                --noc;
+              }
+            }
+          }
+          /* Check for ^Z (anywhere in line) too. */
+          while( iseg < noc ){
+            if( buf[iseg]==0x1a ){
+              conState.stdinEof = 1;
+              noc = iseg; /* Chop ^Z and anything following. */
+              break;
+            }
+            ++iseg;
+          }
+        }else break; /* Drop apparent garbage in. (Could assert.) */
+      }else break;
+    }
+    /* If got nothing, (after ^Z chop), must be at end-of-file. */
+    if( noc == 0 ) return 0;
+    buf[noc] = 0;
+    return buf;
+  }else{
+    return fgets(buf, ncmax, fin);
+  }
+}
+
+# define fgets(b,n,f) utf8_fgets(b,n,f)
+#endif /* SHELL_WIN_UTF8_OPT */
+
 /*
 ** Render output like fprintf().  Except, if the output is going to the
-** console and if this is running on a Windows machine, translate the
-** output from UTF-8 into MBCS.
+** console and if this is running on a Windows machine, and if the -utf8
+** option is unavailable or (available and inactive), translate the
+** output from UTF-8 into MBCS for output through 8-bit stdout stream.
+** (With -utf8 active, no translation is needed and must not be done.)
 */
 #if defined(_WIN32) || defined(WIN32)
 void utf8_printf(FILE *out, const char *zFormat, ...){
   va_list ap;
   va_start(ap, zFormat);
-  if( stdout_is_console && (out==stdout || out==stderr) ){
+  if( stdout_is_console && (out==stdout || out==stderr)
+# if SHELL_WIN_UTF8_OPT
+      && !console_utf8
+# endif
+  ){
     char *z1 = sqlite3_vmprintf(zFormat, ap);
     char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0);
     sqlite3_free(z1);
@@ -620,7 +765,7 @@ static void shell_out_of_memory(void){
 /* Check a pointer to see if it is NULL.  If it is NULL, exit with an
 ** out-of-memory error.
 */
-static void shell_check_oom(void *p){
+static void shell_check_oom(const void *p){
   if( p==0 ) shell_out_of_memory();
 }
 
@@ -799,9 +944,14 @@ static char *local_getline(char *zLine, FILE *in){
     }
   }
 #if defined(_WIN32) || defined(WIN32)
-  /* For interactive input on Windows systems, translate the
-  ** multi-byte characterset characters into UTF-8. */
-  if( stdin_is_interactive && in==stdin ){
+  /* For interactive input on Windows systems, without -utf8,
+  ** translate the multi-byte characterset characters into UTF-8.
+  ** This is the translation that predates the -utf8 option. */
+  if( stdin_is_interactive && in==stdin
+# if SHELL_WIN_UTF8_OPT
+      && !console_utf8
+# endif /* SHELL_WIN_UTF8_OPT */
+  ){
     char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0);
     if( zTrans ){
       i64 nTrans = strlen(zTrans)+1;
@@ -842,10 +992,21 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
 #if SHELL_USE_LOCAL_GETLINE
     printf("%s", zPrompt);
     fflush(stdout);
-    zResult = local_getline(zPrior, stdin);
+    do{
+      zResult = local_getline(zPrior, stdin);
+      zPrior = 0;
+      /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */
+      if( zResult==0 ) sqlite3_sleep(50);
+    }while( zResult==0 && seenInterrupt>0 );
 #else
     free(zPrior);
     zResult = shell_readline(zPrompt);
+    while( zResult==0 ){
+      /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */
+      sqlite3_sleep(50);
+      if( seenInterrupt==0 ) break;
+      zResult = shell_readline("");
+    }
     if( zResult && *zResult ) shell_add_history(zResult);
 #endif
   }
@@ -985,6 +1146,7 @@ static void appendText(ShellText *p, const char *zAppend, char quote){
 */
 static char quoteChar(const char *zName){
   int i;
+  if( zName==0 ) return '"';
   if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
   for(i=0; zName[i]; i++){
     if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
@@ -1366,6 +1528,7 @@ static ShellState shellState;
 #define SHFLG_HeaderSet      0x00000080 /* showHeader has been specified */
 #define SHFLG_DumpDataOnly   0x00000100 /* .dump show data only */
 #define SHFLG_DumpNoSys      0x00000200 /* .dump omits system tables */
+#define SHFLG_TestingMode    0x00000400 /* allow unsafe testing features */
 
 /*
 ** Macros for testing and setting shellFlgs
@@ -1699,6 +1862,7 @@ static void output_quoted_string(FILE *out, const char *z){
   int i;
   char c;
   setBinaryMode(out, 1);
+  if( z==0 ) return;
   for(i=0; (c = z[i])!=0 && c!='\''; i++){}
   if( c==0 ){
     utf8_printf(out,"'%s'",z);
@@ -1828,6 +1992,7 @@ static void output_c_string(FILE *out, const char *z){
 */
 static void output_json_string(FILE *out, const char *z, i64 n){
   unsigned int c;
+  if( z==0 ) z = "";
   if( n<0 ) n = strlen(z);
   fputc('"', out);
   while( n-- ){
@@ -1952,8 +2117,7 @@ static void output_csv(ShellState *p, const char *z, int bSep){
 */
 static void interrupt_handler(int NotUsed){
   UNUSED_PARAMETER(NotUsed);
-  seenInterrupt++;
-  if( seenInterrupt>2 ) exit(1);
+  if( ++seenInterrupt>1 ) exit(1);
   if( globalDb ) sqlite3_interrupt(globalDb);
 }
 
@@ -2774,7 +2938,7 @@ static char *shell_error_context(const char *zSql, sqlite3 *db){
   if( db==0
    || zSql==0
    || (iOffset = sqlite3_error_offset(db))<0
-   || iOffset>=strlen(zSql)
+   || iOffset>=(int)strlen(zSql)
   ){
     return sqlite3_mprintf("");
   }
@@ -3404,10 +3568,14 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){
     sqlite3_bind_text(pQ, 1, zVar, -1, SQLITE_STATIC);
     if( rc==SQLITE_OK && pQ && sqlite3_step(pQ)==SQLITE_ROW ){
       sqlite3_bind_value(pStmt, i, sqlite3_column_value(pQ, 0));
+#ifdef NAN
     }else if( sqlite3_strlike("_NAN", zVar, 0)==0 ){
       sqlite3_bind_double(pStmt, i, NAN);
+#endif
+#ifdef INFINITY
     }else if( sqlite3_strlike("_INF", zVar, 0)==0 ){
       sqlite3_bind_double(pStmt, i, INFINITY);
+#endif
     }else{
       sqlite3_bind_null(pStmt, i);
     }
@@ -3651,7 +3819,7 @@ static void exec_prepared_stmt_columnar(
   azData = sqlite3_malloc64( nAlloc*sizeof(char*) );
   shell_check_oom(azData);
   azNextLine = sqlite3_malloc64( nColumn*sizeof(char*) );
-  shell_check_oom((void*)azNextLine);
+  shell_check_oom(azNextLine);
   memset((void*)azNextLine, 0, nColumn*sizeof(char*) );
   if( p->cmOpts.bQuote ){
     azQuoted = sqlite3_malloc64( nColumn*sizeof(char*) );
@@ -3681,6 +3849,7 @@ static void exec_prepared_stmt_columnar(
     }
     if( wx<0 ) wx = -wx;
     uz = (const unsigned char*)sqlite3_column_name(pStmt,i);
+    if( uz==0 ) uz = (u8*)"";
     azData[i] = translateForDisplayAndDup(uz, &zNotUsed, wx, bw);
   }
   do{
@@ -4614,13 +4783,13 @@ static const char *(azHelp[]) = {
   "        input text.",
 #endif
 #ifndef SQLITE_OMIT_TEST_CONTROL
-  ".imposter INDEX TABLE    Create imposter table TABLE on index INDEX",
+  ",imposter INDEX TABLE    Create imposter table TABLE on index INDEX",
 #endif
   ".indexes ?TABLE?         Show names of indexes",
   "                           If TABLE is specified, only show indexes for",
   "                           tables matching TABLE using the LIKE operator.",
 #ifdef SQLITE_ENABLE_IOTRACE
-  ".iotrace FILE            Enable I/O diagnostic logging to FILE",
+  ",iotrace FILE            Enable I/O diagnostic logging to FILE",
 #endif
   ".limit ?LIMIT? ?VAL?     Display or change the value of an SQLITE_LIMIT",
   ".lint OPTIONS            Report potential schema issues.",
@@ -4729,7 +4898,7 @@ static const char *(azHelp[]) = {
   "   Options:",
   "      --indent             Try to pretty-print the schema",
   "      --nosys              Omit objects whose names start with \"sqlite_\"",
-  ".selftest ?OPTIONS?      Run tests defined in the SELFTEST table",
+  ",selftest ?OPTIONS?      Run tests defined in the SELFTEST table",
   "    Options:",
   "       --init               Create a new SELFTEST table",
   "       -v                   Verbose output",
@@ -4771,9 +4940,9 @@ static const char *(azHelp[]) = {
 #endif
   ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
 #ifndef SQLITE_SHELL_FIDDLE
-  ".testcase NAME           Begin redirecting output to 'testcase-out.txt'",
+  ",testcase NAME           Begin redirecting output to 'testcase-out.txt'",
 #endif
-  ".testctrl CMD ...        Run various sqlite3_test_control() operations",
+  ",testctrl CMD ...        Run various sqlite3_test_control() operations",
   "                           Run \".testctrl\" with no arguments for details",
   ".timeout MS              Try opening locked tables for MS milliseconds",
   ".timer on|off            Turn SQL timer on or off",
@@ -4825,16 +4994,41 @@ static int showHelp(FILE *out, const char *zPattern){
    || cli_strcmp(zPattern,"-all")==0
    || cli_strcmp(zPattern,"--all")==0
   ){
-    /* Show all commands, but only one line per command */
-    if( zPattern==0 ) zPattern = "";
+    enum HelpWanted { HW_NoCull = 0, HW_SummaryOnly = 1, HW_Undoc = 2 };
+    enum HelpHave { HH_Undoc = 2, HH_Summary = 1, HH_More = 0 };
+    /* Show all or most commands
+    ** *zPattern==0   => summary of documented commands only
+    ** *zPattern=='0' => whole help for undocumented commands
+    ** Otherwise      => whole help for documented commands
+    */
+    enum HelpWanted hw = HW_SummaryOnly;
+    enum HelpHave hh = HH_More;
+    if( zPattern!=0 ){
+      hw = (*zPattern=='0')? HW_NoCull|HW_Undoc : HW_NoCull;
+    }
     for(i=0; i<ArraySize(azHelp); i++){
-      if( azHelp[i][0]=='.' || zPattern[0] ){
-        utf8_printf(out, "%s\n", azHelp[i]);
-        n++;
+      switch( azHelp[i][0] ){
+      case ',':
+        hh = HH_Summary|HH_Undoc;
+        break;
+      case '.':
+        hh = HH_Summary;
+        break;
+      default:
+        hh &= ~HH_Summary;
+        break;
+      }
+      if( ((hw^hh)&HH_Undoc)==0 ){
+        if( (hh&HH_Summary)!=0 ){
+          utf8_printf(out, ".%s\n", azHelp[i]+1);
+          ++n;
+        }else if( (hw&HW_SummaryOnly)==0 ){
+          utf8_printf(out, "%s\n", azHelp[i]);
+        }
       }
     }
   }else{
-    /* Look for commands that for which zPattern is an exact prefix */
+    /* Seek documented commands for which zPattern is an exact prefix */
     zPat = sqlite3_mprintf(".%s*", zPattern);
     shell_check_oom(zPat);
     for(i=0; i<ArraySize(azHelp); i++){
@@ -4847,24 +5041,28 @@ static int showHelp(FILE *out, const char *zPattern){
     sqlite3_free(zPat);
     if( n ){
       if( n==1 ){
-        /* when zPattern is a prefix of exactly one command, then include the
-        ** details of that command, which should begin at offset j */
-        while( j<ArraySize(azHelp)-1 && azHelp[j][0]!='.' ){
+        /* when zPattern is a prefix of exactly one command, then include
+        ** the details of that command, which should begin at offset j */
+        while( j<ArraySize(azHelp)-1 && azHelp[j][0]==' ' ){
           utf8_printf(out, "%s\n", azHelp[j]);
           j++;
         }
       }
       return n;
     }
-    /* Look for commands that contain zPattern anywhere.  Show the complete
-    ** text of all commands that match. */
+    /* Look for documented commands that contain zPattern anywhere.
+    ** Show complete text of all documented commands that match. */
     zPat = sqlite3_mprintf("%%%s%%", zPattern);
     shell_check_oom(zPat);
     for(i=0; i<ArraySize(azHelp); i++){
+      if( azHelp[i][0]==',' ){
+        while( i<ArraySize(azHelp)-1 && azHelp[i+1][0]==' ' ) ++i;
+        continue;
+      }
       if( azHelp[i][0]=='.' ) j = i;
       if( sqlite3_strlike(zPat, azHelp[i], 0)==0 ){
         utf8_printf(out, "%s\n", azHelp[j]);
-        while( j<ArraySize(azHelp)-1 && azHelp[j+1][0]!='.' ){
+        while( j<ArraySize(azHelp)-1 && azHelp[j+1][0]==' ' ){
           j++;
           utf8_printf(out, "%s\n", azHelp[j]);
         }
@@ -5190,6 +5388,13 @@ static void open_db(ShellState *p, int openFlags){
     }
     sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0);
 
+    /* Reflect the use or absence of --unsafe-testing invocation. */
+    {
+      int testmode_on = ShellHasFlag(p,SHFLG_TestingMode);
+      sqlite3_db_config(p->db, SQLITE_DBCONFIG_TRUSTED_SCHEMA, testmode_on,0);
+      sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, !testmode_on,0);
+    }
+
 #ifndef SQLITE_OMIT_LOAD_EXTENSION
     sqlite3_enable_load_extension(p->db, 1);
 #endif
@@ -5395,6 +5600,7 @@ static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){
 **    \'    -> '
 **    \\    -> backslash
 **    \NNN  -> ascii character NNN in octal
+**    \xHH  -> ascii character HH in hexadecimal
 */
 static void resolve_backslashes(char *z){
   int i, j;
@@ -5423,6 +5629,15 @@ static void resolve_backslashes(char *z){
         c = '\'';
       }else if( c=='\\' ){
         c = '\\';
+      }else if( c=='x' ){
+        int nhd = 0, hdv;
+        u8 hv = 0;
+        while( nhd<2 && (c=z[i+1+nhd])!=0 && (hdv=hexDigitValue(c))>=0 ){
+          hv = (u8)((hv<<4)|hdv);
+          ++nhd;
+        }
+        i += nhd;
+        c = (u8)hv;
       }else if( c>='0' && c<='7' ){
         c -= '0';
         if( z[i+1]>='0' && z[i+1]<='7' ){
@@ -5522,7 +5737,7 @@ static int sql_trace_callback(
     utf8_printf(p->traceOut, "-- closing database connection\n");
     return 0;
   }
-  if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){
+  if( mType!=SQLITE_TRACE_ROW && pX!=0 && ((const char*)pX)[0]=='-' ){
     zSql = (const char*)pX;
   }else{
     pStmt = (sqlite3_stmt*)pP;
@@ -5554,7 +5769,7 @@ static int sql_trace_callback(
       break;
     }
     case SQLITE_TRACE_PROFILE: {
-      sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
+      sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0;
       utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec);
       break;
     }
@@ -5630,8 +5845,8 @@ static void import_append_char(ImportCtx *p, int c){
 */
 static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
   int c;
-  int cSep = p->cColSep;
-  int rSep = p->cRowSep;
+  int cSep = (u8)p->cColSep;
+  int rSep = (u8)p->cRowSep;
   p->n = 0;
   c = fgetc(p->in);
   if( c==EOF || seenInterrupt ){
@@ -5720,8 +5935,8 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
 */
 static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){
   int c;
-  int cSep = p->cColSep;
-  int rSep = p->cRowSep;
+  int cSep = (u8)p->cColSep;
+  int rSep = (u8)p->cRowSep;
   p->n = 0;
   c = fgetc(p->in);
   if( c==EOF || seenInterrupt ){
@@ -6082,7 +6297,9 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
   if( sqlite3_step(pStmt)==SQLITE_ROW
    && sqlite3_column_bytes(pStmt,0)>100
   ){
-    memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100);
+    const u8 *pb = sqlite3_column_blob(pStmt,0);
+    shell_check_oom(pb);
+    memcpy(aHdr, pb, 100);
     sqlite3_finalize(pStmt);
   }else{
     raw_printf(stderr, "unable to read database header\n");
@@ -6899,7 +7116,10 @@ static int arParseCommand(
       }
     }
   }
-
+  if( pAr->eCmd==0 ){
+    utf8_printf(stderr, "Required argument missing.  Usage:\n");
+    return arUsage(stderr);
+  }
   return SQLITE_OK;
 }
 
@@ -7665,7 +7885,7 @@ FROM (\
       sqlite3_bind_int(pStmt, 1, nDigits);
       rc = sqlite3_step(pStmt);
       sqlite3_finalize(pStmt);
-      assert(rc==SQLITE_DONE);
+      if( rc!=SQLITE_DONE ) rc_err_oom_die(SQLITE_NOMEM);
     }
     assert(db_int(*pDb, zHasDupes)==0); /* Consider: remove this */
     rc = sqlite3_prepare_v2(*pDb, zCollectVar, -1, &pStmt, 0);
@@ -8597,8 +8817,8 @@ static int do_meta_command(char *zLine, ShellState *p){
                            " for import\n");
         goto meta_command_exit;
       }
-      sCtx.cColSep = p->colSeparator[0];
-      sCtx.cRowSep = p->rowSeparator[0];
+      sCtx.cColSep = (u8)p->colSeparator[0];
+      sCtx.cRowSep = (u8)p->rowSeparator[0];
     }
     sCtx.zFile = zFile;
     sCtx.nLine = 1;
@@ -8794,6 +9014,12 @@ static int do_meta_command(char *zLine, ShellState *p){
     int isWO = 0;  /* True if making an imposter of a WITHOUT ROWID table */
     int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */
     int i;
+    if( !ShellHasFlag(p,SHFLG_TestingMode) ){
+      utf8_printf(stderr, ".%s unavailable without --unsafe-testing\n",
+                  "imposter");
+      rc = 1;
+      goto meta_command_exit;
+    }
     if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){
       utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n"
                           "       .imposter off\n");
@@ -8978,7 +9204,8 @@ static int do_meta_command(char *zLine, ShellState *p){
     const char *zFile, *zProc;
     char *zErrMsg = 0;
     failIfSafeMode(p, "cannot run .load in safe mode");
-    if( nArg<2 ){
+    if( nArg<2 || azArg[1][0]==0 ){
+      /* Must have a non-empty FILE. (Will not load self.) */
       raw_printf(stderr, "Usage: .load FILE ?ENTRYPOINT?\n");
       rc = 1;
       goto meta_command_exit;
@@ -10302,28 +10529,36 @@ static int do_meta_command(char *zLine, ShellState *p){
       shell_check_oom(zRevText);
       if( bDebug ) utf8_printf(p->out, "%s\n", zRevText);
       lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0);
-      assert(lrc==SQLITE_OK);
-      if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC);
-      lrc = SQLITE_ROW==sqlite3_step(pStmt);
-      if( lrc ){
-        const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0);
-        sqlite3_stmt *pCheckStmt;
-        lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0);
-        if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery);
-        if( SQLITE_OK==lrc ){
-          if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){
-            double countIrreversible = sqlite3_column_double(pCheckStmt, 0);
-            if( countIrreversible>0 ){
-              int sz = (int)(countIrreversible + 0.5);
-              utf8_printf(stderr,
-                 "Digest includes %d invalidly encoded text field%s.\n",
-                 sz, (sz>1)? "s": "");
+      if( lrc!=SQLITE_OK ){
+        /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the
+        ** user does cruel and unnatural things like ".limit expr_depth 0". */
+        rc = 1;
+      }else{
+        if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC);
+        lrc = SQLITE_ROW==sqlite3_step(pStmt);
+        if( lrc ){
+          const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0);
+          sqlite3_stmt *pCheckStmt;
+          lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0);
+          if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery);
+          if( lrc!=SQLITE_OK ){
+            rc = 1;
+          }else{
+            if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){
+              double countIrreversible = sqlite3_column_double(pCheckStmt, 0);
+              if( countIrreversible>0 ){
+                int sz = (int)(countIrreversible + 0.5);
+                utf8_printf(stderr,
+                     "Digest includes %d invalidly encoded text field%s.\n",
+                            sz, (sz>1)? "s": "");
+              }
             }
+            sqlite3_finalize(pCheckStmt);
           }
-          sqlite3_finalize(pCheckStmt);
+          sqlite3_finalize(pStmt);
         }
-        sqlite3_finalize(pStmt);
       }
+      if( rc ) utf8_printf(stderr, ".sha3sum failed.\n");
       sqlite3_free(zRevText);
     }
 #endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */
@@ -10586,6 +10821,12 @@ static int do_meta_command(char *zLine, ShellState *p){
     int i, n2;
     const char *zCmd = 0;
 
+    if( !ShellHasFlag(p,SHFLG_TestingMode) ){
+      utf8_printf(stderr, ".%s unavailable without --unsafe-testing\n",
+                  "testctrl");
+      rc = 1;
+      goto meta_command_exit;
+    }
     open_db(p, 0);
     zCmd = nArg>=2 ? azArg[1] : "help";
 
@@ -11585,6 +11826,10 @@ static const char zOptions[] =
   "   -stats               print memory stats before each finalize\n"
   "   -table               set output mode to 'table'\n"
   "   -tabs                set output mode to 'tabs'\n"
+  "   -unsafe-testing      allow unsafe commands and modes for testing\n"
+#if SHELL_WIN_UTF8_OPT
+  "   -utf8                setup interactive console code page for UTF-8\n"
+#endif
   "   -version             show SQLite version\n"
   "   -vfs NAME            use NAME as the default VFS\n"
 #ifdef SQLITE_ENABLE_VFSTRACE
@@ -11677,6 +11922,10 @@ static char *cmdline_option_value(int argc, char **argv, int i){
   return argv[i];
 }
 
+static void sayAbnormalExit(void){
+  if( seenInterrupt ) fprintf(stderr, "Program interrupted.\n");
+}
+
 #ifndef SQLITE_SHELL_IS_UTF8
 #  if (defined(_WIN32) || defined(WIN32)) \
    && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__)))
@@ -11697,7 +11946,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   char **argv;
 #endif
 #ifdef SQLITE_DEBUG
-  sqlite3_int64 mem_main_enter = sqlite3_memory_used();
+  sqlite3_int64 mem_main_enter = 0;
 #endif
   char *zErrMsg = 0;
 #ifdef SQLITE_SHELL_FIDDLE
@@ -11718,9 +11967,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   char **argvToFree = 0;
   int argcToFree = 0;
 #endif
-
-  setBinaryMode(stdin, 0);
   setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
+
 #ifdef SQLITE_SHELL_FIDDLE
   stdin_is_interactive = 0;
   stdout_is_console = 1;
@@ -11729,7 +11977,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   stdin_is_interactive = isatty(0);
   stdout_is_console = isatty(1);
 #endif
-
+#if SHELL_WIN_UTF8_OPT
+  atexit(console_restore); /* Needs revision for CLI as library call */
+#endif
+  atexit(sayAbnormalExit);
+#ifdef SQLITE_DEBUG
+  mem_main_enter = sqlite3_memory_used();
+#endif
 #if !defined(_WIN32_WCE)
   if( getenv("SQLITE_DEBUG_BREAK") ){
     if( isatty(0) && isatty(2) ){
@@ -11750,6 +12004,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }
   }
 #endif
+  /* Register a valid signal handler early, before much else is done. */
+#ifdef SIGINT
+  signal(SIGINT, interrupt_handler);
+#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE)
+  if( !SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE) ){
+    fprintf(stderr, "No ^C handler.\n");
+  }
+#endif
 
 #if USE_SYSTEM_SQLITE+0!=1
   if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){
@@ -11789,15 +12051,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   assert( argc>=1 && argv && argv[0] );
   Argv0 = argv[0];
 
-  /* Make sure we have a valid signal handler early, before anything
-  ** else is done.
-  */
-#ifdef SIGINT
-  signal(SIGINT, interrupt_handler);
-#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE)
-  SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE);
-#endif
-
 #ifdef SQLITE_SHELL_DBNAME_PROC
   {
     /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name
@@ -11954,6 +12207,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( cli_strcmp(z,"-nonce")==0 ){
       free(data.zNonce);
       data.zNonce = strdup(argv[++i]);
+    }else if( cli_strcmp(z,"-unsafe-testing")==0 ){
+      ShellSetFlag(&data,SHFLG_TestingMode);
     }else if( cli_strcmp(z,"-safe")==0 ){
       /* no-op - catch this on the second pass */
     }
@@ -12116,6 +12371,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       stdin_is_interactive = 1;
     }else if( cli_strcmp(z,"-batch")==0 ){
       stdin_is_interactive = 0;
+    }else if( cli_strcmp(z,"-utf8")==0 ){
+#if SHELL_WIN_UTF8_OPT
+      console_utf8 = 1;
+#endif /* SHELL_WIN_UTF8_OPT */
     }else if( cli_strcmp(z,"-heap")==0 ){
       i++;
     }else if( cli_strcmp(z,"-pagecache")==0 ){
@@ -12186,6 +12445,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #endif
     }else if( cli_strcmp(z,"-safe")==0 ){
       data.bSafeMode = data.bSafeModePersist = 1;
+    }else if( cli_strcmp(z,"-unsafe-testing")==0 ){
+      /* Acted upon in first pass. */
     }else{
       utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
       raw_printf(stderr,"Use -help for a list of options.\n");
@@ -12193,6 +12454,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }
     data.cMode = data.mode;
   }
+#if SHELL_WIN_UTF8_OPT
+  if( console_utf8 && stdin_is_interactive ){
+    console_prepare();
+  }else{
+    setBinaryMode(stdin, 0);
+    console_utf8 = 0;
+  }
+#endif
 
   if( !readStdin ){
     /* Run all arguments that do not begin with '-' as if they were separate

src/sqlite.h.in

@@ -2398,7 +2398,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_DQS_DML]]
-** <dt>SQLITE_DBCONFIG_DQS_DML</td>
+** <dt>SQLITE_DBCONFIG_DQS_DML</dt>
 ** <dd>The SQLITE_DBCONFIG_DQS_DML option activates or deactivates
 ** the legacy [double-quoted string literal] misfeature for DML statements
 ** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The
@@ -2407,7 +2407,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_DQS_DDL]]
-** <dt>SQLITE_DBCONFIG_DQS_DDL</td>
+** <dt>SQLITE_DBCONFIG_DQS_DDL</dt>
 ** <dd>The SQLITE_DBCONFIG_DQS option activates or deactivates
 ** the legacy [double-quoted string literal] misfeature for DDL statements,
 ** such as CREATE TABLE and CREATE INDEX. The
@@ -2416,7 +2416,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]]
-** <dt>SQLITE_DBCONFIG_TRUSTED_SCHEMA</td>
+** <dt>SQLITE_DBCONFIG_TRUSTED_SCHEMA</dt>
 ** <dd>The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to
 ** assume that database schemas are untainted by malicious content.
 ** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite
@@ -2436,7 +2436,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]]
-** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</td>
+** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</dt>
 ** <dd>The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates
 ** the legacy file format flag.  When activated, this flag causes all newly
 ** created database file to have a schema format version number (the 4-byte
@@ -2445,7 +2445,7 @@ struct sqlite3_mem_methods {
 ** any SQLite version back to 3.0.0 ([dateof:3.0.0]).  Without this setting,
 ** newly created databases are generally not understandable by SQLite versions
 ** prior to 3.3.0 ([dateof:3.3.0]).  As these words are written, there
-** is now scarcely any need to generated database files that are compatible 
+** is now scarcely any need to generate database files that are compatible 
 ** all the way back to version 3.0.0, and so this setting is of little
 ** practical use, but is provided so that SQLite can continue to claim the
 ** ability to generate new database files that are compatible with  version
@@ -2458,23 +2458,35 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]]
-** <dt>SQLITE_DBCONFIG_STMT_SCANSTATUS</td>
+** <dt>SQLITE_DBCONFIG_STMT_SCANSTATUS</dt>
 ** <dd>The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in
 ** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears
 ** a flag that enables collection of the sqlite3_stmt_scanstatus_v2()
 ** statistics. For statistics to be collected, the flag must be set on
 ** the database handle both when the SQL statement is prepared and when it 
 ** is stepped. The flag is set (collection of statistics is enabled)
-** by default.</dd>
+** by default.  This option takes two arguments: an integer and a pointer to
+** an integer..  The first argument is 1, 0, or -1 to enable, disable, or
+** leave unchanged the statement scanstatus option.  If the second argument
+** is not NULL, then the value of the statement scanstatus setting after
+** processing the first argument is written into the integer that the second
+** argument points to.
+** </dd>
 **
 ** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]]
-** <dt>SQLITE_DBCONFIG_REVERSE_SCANORDER</td>
-** <dd>The SQLITE_DBCONFIG_REVERSE_SCANORDER option change the default order
+** <dt>SQLITE_DBCONFIG_REVERSE_SCANORDER</dt>
+** <dd>The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order
 ** in which tables and indexes are scanned so that the scans start at the end
 ** and work toward the beginning rather than starting at the beginning and
-** working toward the end.  Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the
-** same as setting [PRAGMA reverse_unordered_selects].  This configuration option
-** is useful for application testing.</dd>
+** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the
+** same as setting [PRAGMA reverse_unordered_selects].  This option takes
+** two arguments which are an integer and a pointer to an integer.  The first
+** argument is 1, 0, or -1 to enable, disable, or leave unchanged the
+** reverse scan order flag, respectively.  If the second argument is not NULL,
+** then 0 or 1 is written into the integer that the second argument points to
+** depending on if the reverse scan order flag is set after processing the
+** first argument.
+** </dd>
 **
 ** </dl>
 */
@@ -2496,7 +2508,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_VIEW           1015 /* int int* */
 #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT    1016 /* int int* */
 #define SQLITE_DBCONFIG_TRUSTED_SCHEMA        1017 /* int int* */
-#define SQLITE_DBCONFIG_STMT_SCANSTATUS       1080 /* int int*  */
+#define SQLITE_DBCONFIG_STMT_SCANSTATUS       1018 /* int int* */
 #define SQLITE_DBCONFIG_REVERSE_SCANORDER     1019 /* int int* */
 #define SQLITE_DBCONFIG_MAX                   1019 /* Largest DBCONFIG */
 
@@ -6243,6 +6255,13 @@ void sqlite3_activate_cerod(
 ** of the default VFS is not implemented correctly, or not implemented at
 ** all, then the behavior of sqlite3_sleep() may deviate from the description
 ** in the previous paragraphs.
+**
+** If a negative argument is passed to sqlite3_sleep() the results vary by
+** VFS and operating system.  Some system treat a negative argument as an
+** instruction to sleep forever.  Others understand it to mean do not sleep
+** at all. ^In SQLite version 3.42.0 and later, a negative
+** argument passed into sqlite3_sleep() is changed to zero before it is relayed
+** down into the xSleep method of the VFS.
 */
 int sqlite3_sleep(int);
 

src/sqliteInt.h

@@ -281,15 +281,22 @@
 #endif
 
 /*
-** A macro to hint to the compiler that a function should not be
+** Macros to hint to the compiler that a function should or should not be
 ** inlined.
 */
 #if defined(__GNUC__)
 #  define SQLITE_NOINLINE  __attribute__((noinline))
+#  define SQLITE_INLINE    __attribute__((always_inline)) inline
 #elif defined(_MSC_VER) && _MSC_VER>=1310
 #  define SQLITE_NOINLINE  __declspec(noinline)
+#  define SQLITE_INLINE    __forceinline
 #else
 #  define SQLITE_NOINLINE
+#  define SQLITE_INLINE
+#endif
+#if defined(SQLITE_COVERAGE_TEST) || defined(__STRICT_ANSI__)
+# undef SQLITE_INLINE
+# define SQLITE_INLINE
 #endif
 
 /*
@@ -2693,6 +2700,7 @@ struct Index {
                            ** expression, or a reference to a VIRTUAL column */
 #ifdef SQLITE_ENABLE_STAT4
   int nSample;             /* Number of elements in aSample[] */
+  int mxSample;            /* Number of slots allocated to aSample[] */
   int nSampleCol;          /* Size of IndexSample.anEq[] and so on */
   tRowcnt *aAvgEq;         /* Average nEq values for keys not in aSample */
   IndexSample *aSample;    /* Samples of the left-most key */
@@ -4179,6 +4187,7 @@ struct Walker {
     struct CoveringIndexCheck *pCovIdxCk;     /* Check for covering index */
     SrcItem *pSrcItem;                        /* A single FROM clause item */
     DbFixer *pFix;                            /* See sqlite3FixSelect() */
+    Mem *aMem;                                /* See sqlite3BtreeCursorHint() */
   } u;
 };
 
@@ -4448,6 +4457,8 @@ int sqlite3CantopenError(int);
 # define sqlite3Isxdigit(x)  (sqlite3CtypeMap[(unsigned char)(x)]&0x08)
 # define sqlite3Tolower(x)   (sqlite3UpperToLower[(unsigned char)(x)])
 # define sqlite3Isquote(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x80)
+# define sqlite3JsonId1(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x42)
+# define sqlite3JsonId2(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x46)
 #else
 # define sqlite3Toupper(x)   toupper((unsigned char)(x))
 # define sqlite3Isspace(x)   isspace((unsigned char)(x))
@@ -4457,6 +4468,8 @@ int sqlite3CantopenError(int);
 # define sqlite3Isxdigit(x)  isxdigit((unsigned char)(x))
 # define sqlite3Tolower(x)   tolower((unsigned char)(x))
 # define sqlite3Isquote(x)   ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`')
+# define sqlite3JsonId1(x)   (sqlite3IsIdChar(x)&&(x)<'0')
+# define sqlite3JsonId2(x)   sqlite3IsIdChar(x)
 #endif
 int sqlite3IsIdChar(u8);
 

src/test1.c

@@ -2391,6 +2391,7 @@ static int SQLITE_TCLAPI vfsCurrentTimeInt64(
   return TCL_OK;
 }
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
 /*
 ** Usage: create_null_module DB NAME
 */
@@ -2413,6 +2414,7 @@ static int SQLITE_TCLAPI test_create_null_module(
   sqlite3_create_module(db, zName, 0, 0);
   return TCL_OK;
 }
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
 
 #ifdef SQLITE_ENABLE_SNAPSHOT
 /*
@@ -9005,7 +9007,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_register_cksumvfs", test_register_cksumvfs,  0 },
      { "sqlite3_unregister_cksumvfs", test_unregister_cksumvfs,  0 },
      { "number_of_cores",             guess_number_of_cores,     0 },
+#ifndef SQLITE_OMIT_VIRTUALTABLE
      { "create_null_module",       test_create_null_module,     0 },
+#endif
   };
   static int bitmask_size = sizeof(Bitmask)*8;
   static int longdouble_size = sizeof(LONGDOUBLE_TYPE);

src/test_syscall.c

@@ -110,15 +110,15 @@ static int ts_stat(const char *zPath, struct stat *p);
 static int ts_fstat(int fd, struct stat *p);
 static int ts_ftruncate(int fd, off_t n);
 static int ts_fcntl(int fd, int cmd, ... );
-static int ts_read(int fd, void *aBuf, size_t nBuf);
-static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off);
+static ssize_t ts_read(int fd, void *aBuf, size_t nBuf);
+static ssize_t ts_pread(int fd, void *aBuf, size_t nBuf, off_t off);
 /* Note:  pread64() and pwrite64() actually use off64_t as the type on their
 ** last parameter.  But that datatype is not defined on many systems 
 ** (ex: Mac, OpenBSD).  So substitute a likely equivalent: sqlite3_uint64 */
-static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off);
-static int ts_write(int fd, const void *aBuf, size_t nBuf);
-static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off);
-static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off);
+static ssize_t ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off);
+static ssize_t ts_write(int fd, const void *aBuf, size_t nBuf);
+static ssize_t ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off);
+static ssize_t ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off);
 static int ts_fchmod(int fd, mode_t mode);
 static int ts_fallocate(int fd, off_t off, off_t len);
 static void *ts_mmap(void *, size_t, int, int, int, off_t);
@@ -211,7 +211,7 @@ static int tsErrno(const char *zFunc){
 /*
 ** A wrapper around tsIsFail(). If tsIsFail() returns non-zero, set the
 ** value of errno before returning.
-*/ 
+*/
 static int tsIsFailErrno(const char *zFunc){
   if( tsIsFail() ){
     errno = tsErrno(zFunc);
@@ -313,7 +313,7 @@ static int ts_fcntl(int fd, int cmd, ... ){
 /*
 ** A wrapper around read().
 */
-static int ts_read(int fd, void *aBuf, size_t nBuf){
+static ssize_t ts_read(int fd, void *aBuf, size_t nBuf){
   if( tsIsFailErrno("read") ){
     return -1;
   }
@@ -323,7 +323,7 @@ static int ts_read(int fd, void *aBuf, size_t nBuf){
 /*
 ** A wrapper around pread().
 */
-static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){
+static ssize_t ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){
   if( tsIsFailErrno("pread") ){
     return -1;
   }
@@ -333,7 +333,7 @@ static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){
 /*
 ** A wrapper around pread64().
 */
-static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){
+static ssize_t ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){
   if( tsIsFailErrno("pread64") ){
     return -1;
   }
@@ -343,7 +343,7 @@ static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){
 /*
 ** A wrapper around write().
 */
-static int ts_write(int fd, const void *aBuf, size_t nBuf){
+static ssize_t ts_write(int fd, const void *aBuf, size_t nBuf){
   if( tsIsFailErrno("write") ){
     if( tsErrno("write")==EINTR ) orig_write(fd, aBuf, nBuf/2);
     return -1;
@@ -354,7 +354,7 @@ static int ts_write(int fd, const void *aBuf, size_t nBuf){
 /*
 ** A wrapper around pwrite().
 */
-static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){
+static ssize_t ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){
   if( tsIsFailErrno("pwrite") ){
     return -1;
   }
@@ -364,7 +364,7 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){
 /*
 ** A wrapper around pwrite64().
 */
-static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off){
+static ssize_t ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off){
   if( tsIsFailErrno("pwrite64") ){
     return -1;
   }

src/util.c

@@ -670,13 +670,15 @@ int sqlite3Int64ToText(i64 v, char *zOut){
   }
   i = sizeof(zTemp)-2;
   zTemp[sizeof(zTemp)-1] = 0;
-  do{
-    zTemp[i--] = (x%10) + '0';
+  while( 1 /*exit-by-break*/ ){
+    zTemp[i] = (x%10) + '0';
     x = x/10;
-  }while( x );
-  if( v<0 ) zTemp[i--] = '-';
-  memcpy(zOut, &zTemp[i+1], sizeof(zTemp)-1-i);
-  return sizeof(zTemp)-2-i;
+    if( x==0 ) break;
+    i--;
+  };
+  if( v<0 ) zTemp[--i] = '-';
+  memcpy(zOut, &zTemp[i], sizeof(zTemp)-i);
+  return sizeof(zTemp)-1-i;
 }
 
 /*
@@ -841,7 +843,9 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){
       u = u*16 + sqlite3HexToInt(z[k]);
     }
     memcpy(pOut, &u, 8);
-    return (z[k]==0 && k-i<=16) ? 0 : 2;
+    if( k-i>16 ) return 2;
+    if( z[k]!=0 ) return 1;
+    return 0;
   }else
 #endif /* SQLITE_OMIT_HEX_INTEGER */
   {
@@ -877,7 +881,7 @@ int sqlite3GetInt32(const char *zNum, int *pValue){
     u32 u = 0;
     zNum += 2;
     while( zNum[0]=='0' ) zNum++;
-    for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){
+    for(i=0; i<8 && sqlite3Isxdigit(zNum[i]); i++){
       u = u*16 + sqlite3HexToInt(zNum[i]);
     }
     if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){

src/vdbe.c

@@ -2745,7 +2745,7 @@ case OP_IfNullRow: {         /* jump */
   VdbeCursor *pC;
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
-  if( ALWAYS(pC) && pC->nullRow ){
+  if( pC && pC->nullRow ){
     sqlite3VdbeMemSetNull(aMem + pOp->p3);
     goto jump_to_p2;
   }
@@ -3240,7 +3240,7 @@ case OP_Affinity: {
       }else{
         pIn1->u.r = (double)pIn1->u.i;
         pIn1->flags |= MEM_Real;
-        pIn1->flags &= ~MEM_Int;
+        pIn1->flags &= ~(MEM_Int|MEM_Str);
       }
     }
     REGISTER_TRACE((int)(pIn1-aMem), pIn1);

src/vdbe.h

@@ -403,4 +403,8 @@ void sqlite3VdbeScanStatusCounters(Vdbe*, int, int, int);
 void sqlite3VdbePrintOp(FILE*, int, VdbeOp*);
 #endif
 
+#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG)
+int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr);
+#endif
+
 #endif /* SQLITE_VDBE_H */

src/vdbeapi.c

@@ -1337,9 +1337,9 @@ static const void *columnName(
   assert( db!=0 );
   n = sqlite3_column_count(pStmt);
   if( N<n && N>=0 ){
+    u8 prior_mallocFailed = db->mallocFailed;
     N += useType*n;
     sqlite3_mutex_enter(db->mutex);
-    assert( db->mallocFailed==0 );
 #ifndef SQLITE_OMIT_UTF16
     if( useUtf16 ){
       ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]);
@@ -1351,7 +1351,8 @@ static const void *columnName(
     /* A malloc may have failed inside of the _text() call. If this
     ** is the case, clear the mallocFailed flag and return NULL.
     */
-    if( db->mallocFailed ){
+    assert( db->mallocFailed==0 || db->mallocFailed==1 );
+    if( db->mallocFailed > prior_mallocFailed ){
       sqlite3OomClear(db);
       ret = 0;
     }

src/vdbeaux.c

@@ -820,6 +820,8 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
   Op *pOp;
   Parse *pParse = p->pParse;
   int *aLabel = pParse->aLabel;
+
+  assert( pParse->db->mallocFailed==0 ); /* tag-20230419-1 */
   p->readOnly = 1;
   p->bIsReader = 0;
   pOp = &p->aOp[p->nOp-1];
@@ -879,6 +881,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
             ** have non-negative values for P2. */
             assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 );
             assert( ADDR(pOp->p2)<-pParse->nLabel );
+            assert( aLabel!=0 );  /* True because of tag-20230419-1 */
             pOp->p2 = aLabel[ADDR(pOp->p2)];
           }
           break;
@@ -1622,7 +1625,7 @@ VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){
 
 /* Return the most recently added opcode
 */
-VdbeOp * sqlite3VdbeGetLastOp(Vdbe *p){
+VdbeOp *sqlite3VdbeGetLastOp(Vdbe *p){
   return sqlite3VdbeGetOp(p, p->nOp - 1);
 }
 
@@ -5275,6 +5278,20 @@ int sqlite3NotPureFunc(sqlite3_context *pCtx){
   return 1;
 }
 
+#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG)
+/*
+** This Walker callback is used to help verify that calls to
+** sqlite3BtreeCursorHint() with opcode BTREE_HINT_RANGE have 
+** byte-code register values correctly initialized.
+*/
+int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr){
+  if( pExpr->op==TK_REGISTER ){
+    assert( (pWalker->u.aMem[pExpr->iTable].flags & MEM_Undefined)==0 );
+  }
+  return WRC_Continue;
+}
+#endif /* SQLITE_ENABLE_CURSOR_HINTS && SQLITE_DEBUG */
+
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 /*
 ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored
@@ -5337,6 +5354,16 @@ void sqlite3VdbePreUpdateHook(
   PreUpdate preupdate;
   const char *zTbl = pTab->zName;
   static const u8 fakeSortOrder = 0;
+#ifdef SQLITE_DEBUG
+  int nRealCol;
+  if( pTab->tabFlags & TF_WithoutRowid ){
+    nRealCol = sqlite3PrimaryKeyIndex(pTab)->nColumn;
+  }else if( pTab->tabFlags & TF_HasVirtual ){
+    nRealCol = pTab->nNVCol;
+  }else{
+    nRealCol = pTab->nCol;
+  }
+#endif
 
   assert( db->pPreUpdate==0 );
   memset(&preupdate, 0, sizeof(PreUpdate));
@@ -5353,8 +5380,8 @@ void sqlite3VdbePreUpdateHook(
 
   assert( pCsr!=0 );
   assert( pCsr->eCurType==CURTYPE_BTREE );
-  assert( pCsr->nField==pTab->nCol 
-       || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1)
+  assert( pCsr->nField==nRealCol 
+       || (pCsr->nField==nRealCol+1 && op==SQLITE_DELETE && iReg==-1)
   );
 
   preupdate.v = v;

src/vdbemem.c

@@ -157,6 +157,7 @@ int sqlite3VdbeMemValidStrRep(Mem *p){
   char *z;
   int i, j, incr;
   if( (p->flags & MEM_Str)==0 ) return 1;
+  if( p->db && p->db->mallocFailed ) return 1;
   if( p->flags & MEM_Term ){
     /* Insure that the string is properly zero-terminated.  Pay particular
     ** attention to the case where p->n is odd */
@@ -439,7 +440,7 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){
 
   vdbeMemRenderNum(nByte, pMem->z, pMem);
   assert( pMem->z!=0 );
-  assert( pMem->n==sqlite3Strlen30NN(pMem->z) );
+  assert( pMem->n==(int)sqlite3Strlen30NN(pMem->z) );
   pMem->enc = SQLITE_UTF8;
   pMem->flags |= MEM_Str|MEM_Term;
   if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal);
@@ -1483,8 +1484,11 @@ static int valueFromFunction(
   if( pList ) nVal = pList->nExpr;
   assert( !ExprHasProperty(p, EP_IntValue) );
   pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0);
+#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
+  if( pFunc==0 ) return SQLITE_OK;
+#endif
   assert( pFunc );
-  if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 
+  if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0
    || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)
   ){
     return SQLITE_OK;

src/wal.c

@@ -746,19 +746,40 @@ static void walChecksumBytes(
   assert( nByte>=8 );
   assert( (nByte&0x00000007)==0 );
   assert( nByte<=65536 );
+  assert( nByte%4==0 );
 
-  if( nativeCksum ){
-    do {
-      s1 += *aData++ + s2;
-      s2 += *aData++ + s1;
-    }while( aData<aEnd );
-  }else{
+  if( !nativeCksum ){
     do {
       s1 += BYTESWAP32(aData[0]) + s2;
       s2 += BYTESWAP32(aData[1]) + s1;
       aData += 2;
     }while( aData<aEnd );
+  }else if( nByte%64==0 ){
+    do {
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+    }while( aData<aEnd );
+  }else{
+    do {
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+    }while( aData<aEnd );
   }
+  assert( aData==aEnd );
 
   aOut[0] = s1;
   aOut[1] = s2;
@@ -3689,7 +3710,9 @@ int sqlite3WalFrames(
       if( rc ) return rc;
     }
   }
-  assert( (int)pWal->szPage==szPage );
+  if( (int)pWal->szPage!=szPage ){
+    return SQLITE_CORRUPT_BKPT;  /* TH3 test case: cov1/corrupt155.test */
+  }
 
   /* Setup information needed to write frames into the WAL */
   w.pWal = pWal;

src/where.c

@@ -2020,7 +2020,7 @@ static int whereRangeScanEst(
   UNUSED_PARAMETER(pBuilder);
   assert( pLower || pUpper );
 #endif
-  assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 );
+  assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 );
   nNew = whereRangeAdjust(pLower, nOut);
   nNew = whereRangeAdjust(pUpper, nNew);
 
@@ -5982,22 +5982,45 @@ WhereInfo *sqlite3WhereBegin(
   }
   if( pParse->nErr ) goto whereBeginError;
 
-  /* Special case: WHERE terms that do not refer to any tables in the join
-  ** (constant expressions). Evaluate each such term, and jump over all the
-  ** generated code if the result is not true.  
+  /* The False-WHERE-Term-Bypass optimization:
   **
-  ** Do not do this if the expression contains non-deterministic functions
-  ** that are not within a sub-select. This is not strictly required, but
-  ** preserves SQLite's legacy behaviour in the following two cases:
+  ** If there are WHERE terms that are false, then no rows will be output,
+  ** so skip over all of the code generated here.
   **
-  **   FROM ... WHERE random()>0;           -- eval random() once per row
-  **   FROM ... WHERE (SELECT random())>0;  -- eval random() once overall
+  ** Conditions:
+  **
+  **   (1)  The WHERE term must not refer to any tables in the join.
+  **   (2)  The term must not come from an ON clause on the
+  **        right-hand side of a LEFT or FULL JOIN.
+  **   (3)  The term must not come from an ON clause, or there must be
+  **        no RIGHT or FULL OUTER joins in pTabList.
+  **   (4)  If the expression contains non-deterministic functions
+  **        that are not within a sub-select. This is not required
+  **        for correctness but rather to preserves SQLite's legacy
+  **        behaviour in the following two cases:
+  **
+  **          WHERE random()>0;           -- eval random() once per row
+  **          WHERE (SELECT random())>0;  -- eval random() just once overall
+  **
+  ** Note that the Where term need not be a constant in order for this
+  ** optimization to apply, though it does need to be constant relative to
+  ** the current subquery (condition 1).  The term might include variables
+  ** from outer queries so that the value of the term changes from one
+  ** invocation of the current subquery to the next.
   */
   for(ii=0; ii<sWLB.pWC->nBase; ii++){
-    WhereTerm *pT = &sWLB.pWC->a[ii];
+    WhereTerm *pT = &sWLB.pWC->a[ii];  /* A term of the WHERE clause */
+    Expr *pX;                          /* The expression of pT */
     if( pT->wtFlags & TERM_VIRTUAL ) continue;
-    if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){
-      sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL);
+    pX = pT->pExpr;
+    assert( pX!=0 );
+    assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) );
+    if( pT->prereqAll==0                           /* Conditions (1) and (2) */
+     && (nTabList==0 || exprIsDeterministic(pX))   /* Condition (4) */
+     && !(ExprHasProperty(pX, EP_InnerON)          /* Condition (3) */
+          && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 )
+    ){
+      sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL);
       pT->wtFlags |= TERM_CODED;
     }
   }

src/wherecode.c

@@ -1011,11 +1011,12 @@ static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){
 */
 static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
   int rc = WRC_Continue;
+  int reg;
   struct CCurHint *pHint = pWalker->u.pCCurHint;
   if( pExpr->op==TK_COLUMN ){
     if( pExpr->iTable!=pHint->iTabCur ){
-      int reg = ++pWalker->pParse->nMem;   /* Register for column value */
-      sqlite3ExprCode(pWalker->pParse, pExpr, reg);
+      reg = ++pWalker->pParse->nMem;   /* Register for column value */
+      reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg);
       pExpr->op = TK_REGISTER;
       pExpr->iTable = reg;
     }else if( pHint->pIdx!=0 ){
@@ -1023,15 +1024,15 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
       pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn);
       assert( pExpr->iColumn>=0 );
     }
-  }else if( pExpr->op==TK_AGG_FUNCTION ){
-    /* An aggregate function in the WHERE clause of a query means this must
-    ** be a correlated sub-query, and expression pExpr is an aggregate from
-    ** the parent context. Do not walk the function arguments in this case.
-    **
-    ** todo: It should be possible to replace this node with a TK_REGISTER
-    ** expression, as the result of the expression must be stored in a 
-    ** register at this point. The same holds for TK_AGG_COLUMN nodes. */
+  }else if( pExpr->pAggInfo ){
     rc = WRC_Prune;
+    reg = ++pWalker->pParse->nMem;   /* Register for column value */
+    reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg);
+    pExpr->op = TK_REGISTER;
+    pExpr->iTable = reg;
+  }else if( pExpr->op==TK_TRUEFALSE ){
+    /* Do not walk disabled expressions.  tag-20230504-1 */
+    return WRC_Prune;
   }
   return rc;
 }
@@ -1133,7 +1134,7 @@ static void codeCursorHint(
   }
   if( pExpr!=0 ){
     sWalker.xExprCallback = codeCursorHintFixExpr;
-    sqlite3WalkExpr(&sWalker, pExpr);
+    if( pParse->nErr==0 ) sqlite3WalkExpr(&sWalker, pExpr);
     sqlite3VdbeAddOp4(v, OP_CursorHint, 
                       (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0,
                       (const char*)pExpr, P4_EXPR);

src/whereexpr.c

@@ -1211,7 +1211,7 @@ static void exprAnalyze(
      && 0==sqlite3ExprCanBeNull(pLeft)
     ){
       assert( !ExprHasProperty(pExpr, EP_IntValue) );
-      pExpr->op = TK_TRUEFALSE;
+      pExpr->op = TK_TRUEFALSE;  /* See tag-20230504-1 */
       pExpr->u.zToken = "false";
       ExprSetProperty(pExpr, EP_IsFalse);
       pTerm->prereqAll = 0;

src/window.c

@@ -785,6 +785,7 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){
       }
       /* no break */ deliberate_fall_through
 
+    case TK_IF_NULL_ROW:
     case TK_AGG_FUNCTION:
     case TK_COLUMN: {
       int iCol = -1;

test/altertab.test

@@ -981,4 +981,22 @@ do_catchsql_test 32.0 {
   ALTER TABLE t1 RENAME TO x;
 } {1 {error in trigger r1: no tables specified}}
 
+# 2023-04-13 https://sqlite.org/forum/forumpost/ff3840145a
+#
+reset_db
+do_execsql_test 33.0 {
+  CREATE TABLE t1(a TEXT);
+  INSERT INTO t1(a) VALUES('abc'),('def'),(NULL);
+  CREATE TABLE t2(b TEXT);
+  CREATE TRIGGER r3 AFTER INSERT ON t1 BEGIN
+   UPDATE t2 SET (b,a)=(SELECT 1) FROM t1 JOIN t2 ON (SELECT * FROM (SELECT a));
+  END;
+}
+do_catchsql_test 33.1 {
+  ALTER TABLE t1 RENAME COLUMN a TO b;
+} {1 {error in trigger r3 after rename: no such column: a}}
+do_execsql_test 33.2 {
+  SELECT quote(a) FROM t1 ORDER BY +a;
+} {NULL 'abc' 'def'}
+
 finish_test

test/analyze3.test

@@ -736,4 +736,31 @@ do_execsql_test 7.2 {
   ANALYZE sqlite_master;
 }
 
+# 2023-04-22 https://sqlite.org/forum/info/6c118daad0f1f5ef
+# Case differences in the sqlite_stat4.idx field should not matter.
+#
+reset_db
+do_execsql_test 8.0 {
+  CREATE TABLE t1(a PRIMARY KEY, v) WITHOUT ROWID;
+  ANALYZE sqlite_schema;
+  INSERT INTO sqlite_stat1 VALUES('t1','t1','1 1');
+  INSERT INTO sqlite_stat4 VALUES('t1','t1','1','0','0',X'021b76657273696f6e');
+  INSERT INTO sqlite_stat4 VALUES('T1','T1','1','0','0',X'021b76657273696f6e');
+  ANALYZE sqlite_schema;
+} {}
+
+# 2023-05-03 https://sqlite.org/forum/forumpost/537d8ab118
+# Same index appears by two different names in the sqlite_stat4 table.
+#
+reset_db
+do_execsql_test 8.1 {
+  CREATE TABLE t1(a INT PRIMARY KEY, b INT) WITHOUT ROWID;
+  ANALYZE sqlite_schema;
+  INSERT INTO sqlite_stat4 VALUES
+     ('t1','t1','1','2','2',X'03000103'),
+     ('t1','sqlite_autoindex_t1_1','1','2','2',X'03000103');
+  ANALYZE sqlite_schema;
+  PRAGMA integrity_check;
+} {ok}
+
 finish_test

test/backup.test

@@ -977,5 +977,28 @@ do_test backup-11.1 {
   sqlite3_backup B db1 main db2 temp
   B finish
 } {SQLITE_OK}
+db1 close
+db2 close
+
+#-------------------------------------------------------------------------
+do_test backup-12.1 {
+  sqlite3 db1 :memory:
+  sqlite3 db2 :memory:
+  db1 eval {
+    PRAGMA page_size = 8192;
+    CREATE TABLE t1(x);
+  }
+  db2 eval {
+    PRAGMA page_size = 1024;
+    CREATE TABLE t2(x);
+  }
+
+  sqlite3_backup B db1 main db2 temp
+  B step 100
+  B finish
+} {SQLITE_READONLY}
+
+
+
 
 finish_test

test/cursorhint.test

@@ -180,4 +180,35 @@ do_execsql_test 5.1 {
     ORDER BY 1;
 }
 
+# 2023-04-10 https://sqlite.org/forum/forumpost/0b53708c95
+#
+do_execsql_test 6.0 {
+  CREATE TABLE t6(a TEXT UNIQUE, b TEXT);
+  INSERT INTO t6(a,b) VALUES('uvw','xyz'),('abc','def');
+  WITH v1(a) AS (SELECT a COLLATE NOCASE FROM t6)
+  SELECT v1.a, count(*) FROM t6 LEFT JOIN v1 ON true
+   GROUP BY 1
+  HAVING (SELECT true FROM t6 AS aa LEFT JOIN t6 AS bb ON length(v1.a)>5);
+} {abc 2 uvw 2}
+
+
+# 2023-05-04 https://sqlite.org/forum/forumpost/29a47cf6d1
+#
+# codeCursorHint() should not walk expressions that have been optimized
+# out and converted to TRUE or FALSE.  This only comes up when compiling
+# with SQLITE_ENABLE_CURSOR_HINTS
+#
+reset_db
+do_execsql_test 7.1 {
+  CREATE TABLE t1(a INT PRIMARY KEY) WITHOUT ROWID;
+  CREATE TABLE t2(b INT PRIMARY KEY) WITHOUT ROWID;
+  CREATE TABLE t3(c INT PRIMARY KEY) WITHOUT ROWID;
+  INSERT INTO t1(a) VALUES(1),(2);
+  INSERT INTO t2(b) VALUES(4),(8);
+  INSERT INTO t3(c) VALUES(16),(32);
+  CREATE VIEW v4(d) AS SELECT c FROM t3;
+  SELECT * FROM t1 RIGHT JOIN t2 ON true JOIN v4 ON (d IS NULL);
+} {}
+
+
 finish_test

test/fkey1.test

@@ -272,4 +272,15 @@ do_catchsql_test 8.3 {
   REINDEX;
 } {1 {database disk image is malformed}}
 
+# 2023-04-13 https://bugs.chromium.org/p/chromium/issues/detail?id=1405220
+# Avoid double-de-quoting of table names when processing foreign keys.
+#
+reset_db
+do_execsql_test 9.1 {
+  PRAGMA foreign_keys = ON;
+  CREATE TABLE """1"("""2", """3" PRIMARY KEY);
+  CREATE TABLE """4"("""5" REFERENCES """1" ON DELETE RESTRICT);
+  DELETE FROM """1";
+}
+
 finish_test

test/indexexpr1.test

@@ -591,6 +591,29 @@ do_execsql_test indexexpr1-2140 {
   SELECT b FROM t1;
 } 400
 
-
+# 2023-04-18 Forum post https://sqlite.org/forum/forumpost/f34e32d120 from
+# Alexis King.
+#
+# This problem originates at check-in b9190d3da70c4171 (2022-11-25).
+# A similar problem arose on 2023-03-04 at
+# https://sqlite.org/forum/forumpost/a68313d054 and was fixed at
+# check-in e06973876993926f.  See the test case tkt-99378-400.
+# 
+reset_db
+do_execsql_test indexexpr1-2200 {
+  CREATE TABLE t1(id INTEGER PRIMARY KEY, tag INT);
+  INSERT INTO t1 VALUES (0, 7), (1, 8);
+  CREATE TABLE t2(type INT, t1_id  INT, value  INT);
+  INSERT INTO t2 VALUES (0, 0, 100), (0, 1, 101);
+  CREATE INDEX t1x ON t1(-tag);
+  SELECT u.tag, v.max_value
+    FROM (SELECT tag FROM t1 GROUP BY -tag) u
+    JOIN (SELECT t1.tag AS "tag", t2.type AS "type",
+                 MAX(t2.value) AS "max_value"
+            FROM t1
+                 JOIN t2 ON t2.t1_id = t1.id
+           GROUP BY t2.type, t1.tag
+         ) v ON v.type = 0 AND v.tag = u.tag;
+} {7 100 8 101}
 
 finish_test

test/join.test

@@ -1245,4 +1245,22 @@ do_eqp_test join-28.2 {
 #   |--SCAN t4
 #   `--SEARCH t3 USING AUTOMATIC COVERING INDEX (a=?)
 
+
+# 2023-05-01 https://sqlite.org/forum/forumpost/96cd4a7e9e
+#
+reset_db
+db null NULL
+do_execsql_test join-29.1 {
+  CREATE TABLE t0(a INT); INSERT INTO t0(a) VALUES (1);
+  CREATE TABLE t1(b INT); INSERT INTO t1(b) VALUES (2);
+  CREATE VIEW v2(c) AS SELECT 3 FROM t1;
+  SELECT * FROM t1 JOIN v2 ON 0     FULL OUTER JOIN t0 ON true;
+} {NULL NULL 1}
+do_execsql_test join-29.2 {
+  SELECT * FROM t1 JOIN v2 ON 1=0   FULL OUTER JOIN t0 ON true;
+} {NULL NULL 1}
+do_execsql_test join-29.3 {
+  SELECT * FROM t1 JOIN v2 ON false FULL OUTER JOIN t0 ON true;
+} {NULL NULL 1}
+
 finish_test

test/joinH.test

@@ -89,5 +89,35 @@ do_execsql_test 4.4 {
   SELECT (d IS NULL) FROM t1 RIGHT JOIN t2 ON (j=33);
 } {1}
 
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 5.0 {
+  CREATE TABLE t0(w);
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(y);
+  CREATE TABLE t3(z);
+  INSERT INTO t3 VALUES('t3val');
+}
+
+do_execsql_test 5.1 {
+  SELECT * FROM t1 INNER JOIN t2 ON (0) RIGHT OUTER JOIN t3;
+} {{} {} t3val}
+
+do_execsql_test 5.2 {
+  SELECT * FROM t1 INNER JOIN t2 ON (0) FULL OUTER JOIN t3;
+} {{} {} t3val}
+
+do_execsql_test 5.3 {
+  SELECT * FROM t3 LEFT JOIN t2 ON (0);
+} {t3val {}}
+
+do_execsql_test 5.4 {
+  SELECT * FROM t0 RIGHT JOIN t1 INNER JOIN t2 ON (0) RIGHT JOIN t3
+} {{} {} {} t3val}
+
+do_execsql_test 5.5 {
+  SELECT * FROM t0 RIGHT JOIN t1 INNER JOIN t2 ON (0)
+} {}
 
 finish_test

test/json/README.md

@@ -0,0 +1,27 @@
+The files in this subdirectory are used to help measure the performance
+of the SQLite JSON parser.
+
+# 1.0 Prerequisites
+
+  1.  Valgrind
+
+  2.  Fossil
+
+# 2.0 Setup
+
+  1.  Run: "`tclsh json-generator.tcl | sqlite3 json100mb.db`" to create
+      the 100 megabyte test database.  Do this so that the "json100mb.db"
+      file lands in the directory from which you will run tests, not in
+      the test/json subdirectory of the source tree.
+
+  2.  Build the baseline sqlite3.c file.  ("`make sqlite3.c`")
+
+  3.  Run "`sh json-speed-check-1.sh trunk`".   This creates the baseline
+      profile in "jout-trunk.txt".
+
+# 3.0 Testing
+
+  1.  Build the sqlite3.c to be tested.
+
+  2.  Run "`sh json-speed-check-1.sh x1`".  The profile output will appear
+      in jout-x1.txt.  Substitute any label you want in place of "x1".

test/json/json-generator.tcl

@@ -0,0 +1,401 @@
+#!/usr/bin/tclsh
+#
+# Generate SQL that will populate an SQLite database with about 100 megabytes
+# of pseudo-random JSON text.
+#
+#     tclsh json-generator.tcl | sqlite3 json110mb.db
+#
+# srand() is used to initialize the random seed so that the same JSON
+# is generated for every run.
+#
+expr srand(12345678)
+set wordlist {
+   ability   able      abroad    access    account   act
+   action    active    actor     add       address   adept
+   adroit    advance   advice    affect    age       ageless
+   agency    agent     agile     agree     air       airfare
+   airline   airport   alert     almond    alpha     always
+   amend     amount    amplify   analyst   anchor    angel
+   angelic   angle     ankle     annual    answer    antique
+   anybody   anyhow    appeal    apple     apricot   apt
+   area      argon     arm       army      arrival   arsenic
+   art       artful    article   arugula   aside     ask
+   aspect    assist    assume    atom      atone     attempt
+   author    autumn    average   avocado   award     awl
+   azure     back      bacon     bag       bagel     bake
+   baker     balance   ball      balloon   bamboo    banana
+   band      banjo     bank      barium    base      basil
+   basin     basis     basket    bass      bat       bath
+   battery   beach     beak      bean      bear      bearcub
+   beauty    beef      beet      beige     being     bell
+   belly     belt      bench     bend      benefit   best
+   beta      better    beyond    bicycle   bid       big
+   bike      bill      bird      biscuit   bismuth   bisque
+   bit       black     blank     blest     blind     bliss
+   block     bloom     blue      board     boat      body
+   bokchoy   bone      bonus     book      bookish   boot
+   border    boron     boss      bossy     bottle    bottom
+   bow       bowl      bowtie    box       brain     brainy
+   branch    brave     bravely   bread     break     breath
+   breezy    brick     bridge    brie      brief     briefly
+   bright    broad     broil     bromine   bronze    brother
+   brow      brown     brush     buddy     budget    buffalo
+   bug       bugle     bull      bunch     burger    burly
+   burrito   bus       busy      butter    button    buy
+   buyer     byte      cab       cabbage   cabinet   cable
+   cadet     cadmium   caesium   cake      calcium   caliper
+   call      caller    calm      calmly    camera    camp
+   can       canary    cancel    candle    candy     cap
+   capable   caper     capital   captain   car       carbon
+   card      care      career    careful   carp      carpet
+   carrot    carry     case      cash      cassava   casual
+   cat       catch     catfish   catsear   catsup    cause
+   cave      celery    cell      century   chain     chair
+   chalk     chance    change    channel   chapter   chard
+   charge    charity   chart     check     cheddar   cheery
+   cheese    chicken   chicory   chiffon   child     chin
+   chip      chives    choice    chowder   chum      church
+   circle    city      claim     clam      class     classic
+   classy    clay      clean     cleaner   clear     clearly
+   clerk     click     client    climate   clock     clorine
+   closet    clothes   cloud     clown     club      clue
+   cluster   coach     coast     coat      cobbler   cobolt
+   cod       code      coffee    colby     cold      collar
+   college   comb      combine   comet     comfort   command
+   comment   common    company   complex   concept   concern
+   concert   conduit   consist   contact   contest   context
+   control   convert   cook      cookie    copilot   copper
+   copy      coral     cordial   corn      corner    corny
+   correct   cost      count     counter   country   county
+   couple    courage   course    court     cover     cow
+   cowbird   crab      crack     craft     crash     crazy
+   cream     credit    creek     cress     crevice   crew
+   crimson   croaker   crop      cross     crowd     cube
+   cuckoo    cuisine   culture   cup       current   curve
+   cut       cyan      cycle     dagger    daily     dance
+   dare      darter    data      date      day       daylily
+   deal      dear      dearly    debate    debit     decade
+   decimal   deep      deft      deftly    degree    delay
+   deluxe    deposit   depth     design    desk      detail
+   device    dew       diamond   diet      dig       dill
+   dinner    dip       direct    dirt      dish      disk
+   display   diver     divide    divine    doctor    dodger
+   donut     door      dot       double    dough     draft
+   drag      dragon    drama     draw      drawer    drawing
+   dream     drill     drink     drive     driver    drop
+   drum      dry       dryer     drywall   duck      due
+   dump      dusk      dust      duty      dye       eagle
+   ear       earring   earth     ease      east      easy
+   eat       economy   edge      editor    eel       effect
+   effort    egg       eight     elbow     elegant   element
+   elf       elk       email     emerald   employ    end
+   endive    endless   energy    engine    enjoy     enter
+   entry     equal     equip     error     escape    essay
+   eternal   evening   event     exam      example   excuse
+   exit      expert    extent    extreme   eye       face
+   fact      factor    factual   fail      failure   fair
+   fajita    fall      family    fan       fang      farm
+   farmer    fat       fault     feature   feed      feel
+   feeling   fench     fennel    festive   few       fiber
+   field     fig       figure    file      fill      film
+   filter    final     finance   finding   finger    finish
+   fire      fish      fishing   fit       fitting   five
+   fix       flier     flight    floor     floral    florine
+   flour     flow      flower    fly       flying    focus
+   fold      folding   food      foot      force     forest
+   forever   forgive   form      formal    format    fortune
+   forum     frame     free      freedom   freely    fresh
+   friend    frog      front     fruit     fuchsia   fuel
+   fun       funny     future    gain      galaxy    gallium
+   game      gamma     gap       garage    garden    garlic
+   gas       gate      gather    gauge     gear      gem
+   gene      general   gentle    gently    gherkin   ghost
+   gift      give      glad      glass     gleeful   glossy
+   glove     glue      goal      goat      goby      gold
+   goldeye   golf      good      gouda     goulash   gourd
+   grab      grace     grade     gram      grand     grape
+   grapes    grass     gravy     gray      great     green
+   grits     grocery   ground    group     grouper   grout
+   growth    guard     guave     guess     guest     guide
+   guitar    gumbo     guppy     habit     hacksaw   haddock
+   hafnium   hagfish   hair      half      halibut   hall
+   hammer    hand      handle    handy     hanger    happy
+   hat       havarti   hay       haybale   head      health
+   healthy   hearing   heart     hearty    heat      heavy
+   heel      height    helium    hello     help      helpful
+   herald    herring   hide      high      highly    highway
+   hill      hip       hipster   hire      history   hit
+   hoki      hold      hole      holiday   holly     home
+   honest    honey     hook      hope      hopeful   horizon
+   horn      horse     host      hotel     hour      house
+   housing   human     humane    humor     hunt      hurry
+   ice       icecube   icefish   icy       idea      ideal
+   image     impact    impress   inch      income    indigo
+   initial   inkpen    insect    inside    intense   invite
+   iodine    iridium   iron      island    issue     item
+   ivory     jacket    jargon    javelin   jello     jelly
+   jewel     job       jocund    join      joint     joke
+   jovial    joy       joyful    joyous    judge     juice
+   jump      junior    jury      just      justice   kale
+   keel      keep      kelp      ketchup   key       keyhole
+   keyway    khaki     kick      kid       kidney    kiloohm
+   kind      kindly    king      kitchen   kite      kiwi
+   knee      knife     krill     krypton   kumquat   lab
+   lace      lack      ladder    lake      lamp      lamprey
+   land      laser     laugh     law       lawn      lawyer
+   layer     lead      leader    leading   leaf      leafy
+   league    leather   leave     lecture   leek      leg
+   lemon     length    lentil    lesson    let       letter
+   lettuce   level     library   life      lift      light
+   lily      lime      limit     line      linen     link
+   lip       list      listen    lithium   lively    living
+   lizard    load      loan      lobster   local     lock
+   log       long      longfin   look      lotus     love
+   lovely    loving    low       lucid     luck      luffa
+   lunch     lung      machine   magenta   magnet    mail
+   main      major     make      mall      manager   mango
+   manner    many      map       march     market    maroon
+   martian   master    match     math      matter    maximum
+   maybe     meal      meaning   meat      media     medium
+   meet      meeting   melody    melon     member    memory
+   mention   menu      mercury   merry     mess      message
+   messy     metal     meter     method    micron    middle
+   might     mile      milk      mind      mine      minimum
+   minnow    minor     mint      minute    mirror    miss
+   mission   misty     mix       mixer     mixture   mobile
+   mode      model     moment    monitor   monk      month
+   moon      moray     morning   most      motor     mouse
+   mouth     move      mover     movie     much      mud
+   mudfish   muffin    mullet    munster   muon      muscle
+   music     mustard   nail      name      nation    native
+   natural   nature    navy      neat      neatly    nebula
+   neck      needle    neon      nerve     net       network
+   neutron   news      nibble    nice      nickel    night
+   niobium   nobody    noise     noodle    normal    north
+   nose      note      nothing   notice    nova      novel
+   number    nurse     nursery   oar       object    offer
+   office    officer   oil       okay      okra      old
+   olive     one       onion     open      opening   opinion
+   option    orange    orbit     orchid    order     oregano
+   other     ounce     outcome   outside   oven      owner
+   oxygen    oyster    pace      pack      package   page
+   pager     paint     pair      pale      pan       pancake
+   papaya    paper     pardon    parent    park      parking
+   parsley   parsnip   part      partner   party     pass
+   passage   past      pasta     path      patient   pattern
+   pause     pay       pea       peace     peach     peacock
+   peahen    peak      peanut    pear      pearl     pen
+   penalty   pencil    pension   people    pepper    perch
+   perfect   period    permit    person    phase     phone
+   photo     phrase    physics   piano     pick      picture
+   pie       piece     pigeon    pike      pilot     pin
+   pink      pinkie    pious     pipe      pitch     pizza
+   place     plan      plane     planet    plant     planter
+   plastic   plate     play      player    playful   plenty
+   pliers    plum      pod       poem      poet      poetry
+   point     police    policy    pollock   pony      pool
+   pop       popover   poptart   pork      port      portal
+   post      pot       potato    pound     powder    power
+   present   press     price     pride     primary   print
+   prior     private   prize     problem   process   produce
+   product   profile   profit    program   project   promise
+   prompt    proof     proper    protein   proton    public
+   puff      puffer    pull      pumpkin   pup       pupfish
+   pure      purple    purpose   push      put       quality
+   quark     quarter   quiet     quill     quit      quote
+   rabbit    raccoon   race      radiant   radio     radish
+   radium    radon     rain      rainbow   raise     ramp
+   ranch     range     rasp      rate      ratio     ray
+   razor     reach     read      reading   real      reality
+   reason    recipe    record    recover   red       redeem
+   reed      reef      refuse    region    regret    regular
+   relaxed   release   relief    relish    remote    remove
+   rent      repair    repeat    reply     report    request
+   reserve   resist    resolve   resort    rest      result
+   return    reveal    review    reward    ribbon    rice
+   rich      ride      ridge     right     ring      rise
+   risk      river     rivet     road      roast     rock
+   rocket    role      roll      roof      room      rope
+   rose      rough     roughy    round     row       royal
+   rub       ruby      rudder    ruin      rule      run
+   runner    rush      rust      sacred    saddle    safe
+   safety    sail      salad     salami    sale      salmon
+   salt      sample    sand      sander    sandy     sauce
+   save      saving    saw       scale     scampi    scene
+   scheme    school    score     screen    script    sea
+   search    season    seat      second    secret    sector
+   seemly    self      sell      senate    senior    sense
+   series    serve     set       shake     shape     share
+   shark     shell     shift     shine     shiny     ship
+   shock     shoe      shoot     shop      shovel    show
+   side      sign      signal    silk      silly     silver
+   simple    sing      singer    single    sink      site
+   size      skill     skin      sky       slate     sleep
+   sleepy    slice     slide     slip      smart     smell
+   smelt     smile     smoke     smooth    snap      snipe
+   snow      snowy     sock      socket    sodium    soft
+   softly    soil      sole      solid     song      sorrel
+   sort      soul      sound     soup      source    south
+   space     spare     speech    speed     spell     spend
+   sphere    spice     spider    spirit    spite     split
+   spoon     sport     spot      spray     spread    spring
+   squab     square    squash    stable    staff     stage
+   stand     staple    star      start     state     status
+   stay      steak     steel     step      stern     stew
+   stick     still     stock     stone     stop      store
+   storm     story     strain    street    stress    strike
+   string    stroke    strong    studio    study     stuff
+   style     sugar     suit      sulfur    summer    sun
+   sunny     sunset    super     superb    surf      survey
+   sweet     swim      swing     switch    symbol    system
+   table     tackle    tail      tale      talk      tan
+   tank      tap       tape      target    task      taste
+   tau       tea       teach     teal      team      tear
+   tell      ten       tender    tennis    tent      term
+   test      tetra     text      thanks    theme     theory
+   thing     think     thread    throat    thumb     ticket
+   tidy      tie       tiger     till      time      timely
+   tin       tip       title     toast     today     toe
+   tomato    tone      tongue    tool      tooth     top
+   topic     total     touch     tough     tour      towel
+   tower     town      track     trade     train     trash
+   travel    tray      treat     tree      trick     trip
+   trout     trowel    truck     trupet    trust     truth
+   try       tube      tuna      tune      turf      turkey
+   turn      turnip    tutor     tux       tweet     twist
+   two       type      union     unique    unit      upbeat
+   upper     use       useful    user      usual     valley
+   value     van       vase      vast      veil      vein
+   velvet    verse     very      vessel    vest      video
+   view      violet    visit     visual    vivid     voice
+   volume    vowel     voyage    waffle    wait      wake
+   walk      wall      warm      warmth    wasabi    wash
+   watch     water     wave      wax       way       wealth
+   wear      web       wedge     week      weekly    weight
+   west      whale     what      wheat     wheel     when
+   where     while     who       whole     why       will
+   win       wind      window    wing      winner    winter
+   wire      wish      witty     wolf      wonder    wood
+   wool      woolly    word      work      worker    world
+   worry     worth     worthy    wrap      wrench    wrist
+   writer    xenon     yak       yam       yard      yarrow
+   year      yearly    yellow    yew       yogurt    young
+   youth     zebra     zephyr    zinc      zone      zoo
+}
+set nwordlist [llength $wordlist]
+
+proc random_char {} {
+  return [string index \
+             "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" \
+             [expr {int(rand()*52)}]]
+}
+proc random_label {} {
+  set label [random_char]
+  while {rand()>0.8} {
+    append label [random_char]
+  }
+  if {rand()>0.9} {append label -}
+  append label [format %d [expr {int(rand()*100)}]]
+  return $label
+}
+proc random_numeric {} {
+  set n [expr {(rand()*2-1.0)*1e6}]
+  switch [expr {int(rand()*6)}] {
+    0  {set format %.3f}
+    1  {set format %.6E}
+    2  {set format %.4e}
+    default  {set format %g}
+  }
+  return [format $format $n]
+}
+
+
+proc random_json {limit indent} {
+  global nwordlist wordlist
+  set res {}
+  if {$indent==0 || ($limit>0 && rand()>0.5)} {
+    incr limit -1
+    incr indent 2
+    set n [expr {int(rand()*5)+1}]
+    if {$n==5} {incr n [expr {int(rand()*10)}]}
+    if {rand()>0.5} {
+      set res \173\n
+      for {set i 0} {$i<$n} {incr i} {
+        append res [string repeat { } $indent]
+        if {rand()>0.8} {
+          if {rand()>0.5} {
+            set sep ":\n   [string repeat { } $indent]"
+          } else {
+            set sep " : "
+          }
+        } else {
+          set sep :
+        }
+        append res \"[random_label]\"$sep[random_json $limit $indent]
+        if {$i<$n-1} {append res ,}
+        append res \n
+      }
+      incr indent -2
+      append res [string repeat { } $indent]
+      append res \175
+      return $res
+    } else {
+      set res \[\n
+      for {set i 0} {$i<$n} {incr i} {
+        append res [string repeat { } $indent]
+        append res [random_json $limit $indent]
+        if {$i<$n-1} {append res ,}
+        append res \n
+      }
+      incr indent -2
+      append res [string repeat { } $indent]
+      append res \]
+      return $res
+    }
+  } elseif {rand()>0.9} {
+    if {rand()>0.7} {return "true"}
+    if {rand()>0.5} {return "false"}
+    return "null"
+  } elseif {rand()>0.5} {
+    return [random_numeric]
+  } else {
+    set res \"
+    set n [expr {int(rand()*4)+1}]
+    if {$n>=4} {set n [expr {$n+int(rand()*6)}]}
+    for {set i 0} {$i<$n} {incr i} {
+      if {rand()<0.05} {
+        set w [random_numeric]
+      } else {
+        set k [expr {int(rand()*$nwordlist)}]
+        set w [lindex $wordlist $k]
+      }
+      if {rand()<0.07} {
+         set w \\\"$w\\\"
+      }
+      if {$i<$n-1} {
+        switch [expr {int(rand()*9)}] {
+          0       {set sp {, }}
+          1       {set sp "\\n "}
+          2       {set sp "-"}
+          default {set sp { }}
+        }
+        append res $w$sp
+      } else {
+        append res $w
+        if {rand()<0.2} {append res .}
+      }
+    }
+    return $res\"
+  }
+}
+
+puts "CREATE TABLE IF NOT EXISTS data1(x JSON);"
+puts "BEGIN;"
+set sz 0
+for {set i 0} {$sz<100000000} {incr i} {
+  set j [random_json 7 0]
+  incr sz [string length $j]
+  puts "INSERT INTO data1(x) VALUES('$j');"
+}
+puts "COMMIT;"
+puts "SELECT sum(length(x)) FROM data1;"

test/json/json-q1.txt

@@ -0,0 +1,4 @@
+.mode qbox
+.timer on
+.param set $label 'q87'
+SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL;

test/json/json-speed-check.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# This is a template for a script used for day-to-day size and 
+# performance monitoring of SQLite.  Typical usage:
+#
+#     sh speed-check.sh trunk  #  Baseline measurement of trunk
+#     sh speed-check.sh x1     # Measure some experimental change
+#     fossil xdiff --tk jout-trunk.txt jout-x1.txt   # View chanages
+#
+# There are multiple output files, all with a base name given by
+# the first argument:
+#
+#     summary-$BASE.txt           # Copy of standard output
+#     jout-$BASE.txt              # cachegrind output
+#     explain-$BASE.txt           # EXPLAIN listings (only with --explain)
+#
+if test "$1" = ""
+then
+  echo "Usage: $0 OUTPUTFILE [OPTIONS]"
+  exit
+fi
+NAME=$1
+shift
+#CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
+CC_OPTS="-DSQLITE_ENABLE_MEMSYS5"
+CC=gcc
+LEAN_OPTS="-DSQLITE_THREADSAFE=0"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_MEMSTATUS=0"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOBS"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_MAX_EXPR_DEPTH=0"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DECLTYPE"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DEPRECATED"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA"
+BASELINE="trunk"
+doExplain=0
+doCachegrind=1
+doVdbeProfile=0
+doWal=1
+doDiff=1
+while test "$1" != ""; do
+  case $1 in
+    --nodiff)
+	doDiff=0
+        ;;
+    --lean)
+        CC_OPTS="$CC_OPTS $LEAN_OPTS"
+        ;;
+    --clang)
+        CC=clang
+        ;;
+    --gcc7)
+        CC=gcc-7
+        ;;
+    -*)
+        CC_OPTS="$CC_OPTS $1"
+        ;;
+    *)
+	BASELINE=$1
+        ;;
+  esac
+  shift
+done
+echo "NAME           = $NAME" | tee summary-$NAME.txt
+echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
+rm -f cachegrind.out.* jsonshell
+$CC -g -Os -Wall -I. $CC_OPTS ./shell.c ./sqlite3.c -o jsonshell -ldl -lpthread
+ls -l jsonshell | tee -a summary-$NAME.txt
+home=`echo $0 | sed -e 's,/[^/]*$,,'`
+echo ./jsonshell json100mb.db "<$home/json-q1.txt"
+valgrind --tool=cachegrind ./jsonshell json100mb.db <$home/json-q1.txt \
+      2>&1 | tee -a summary-$NAME.txt
+cg_anno.tcl cachegrind.out.* >jout-$NAME.txt
+echo '*****************************************************' >>jout-$NAME.txt
+sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>jout-$NAME.txt
+if test "$NAME" != "$BASELINE"; then
+  fossil xdiff --tk -c 20 jout-$BASELINE.txt jout-$NAME.txt
+fi

test/json101.test

@@ -310,12 +310,33 @@ do_execsql_test json-6.1 {
   SELECT json_valid('{"a":55,"b":72,}');
 } {0}
 do_execsql_test json-6.2 {
+  SELECT json_error_position('{"a":55,"b":72,}');
+} {0}
+do_execsql_test json-6.3 {
+  SELECT json_valid(json('{"a":55,"b":72,}'));
+} {1}
+do_execsql_test json-6.4 {
+  SELECT json_valid('{"a":55,"b":72 , }');
+} {0}
+do_execsql_test json-6.5 {
+  SELECT json_error_position('{"a":55,"b":72 , }');
+} {0}
+do_execsql_test json-6.6 {
+  SELECT json_error_position('{"a":55,"b":72,,}');
+} {16}
+do_execsql_test json-6.7 {
   SELECT json_valid('{"a":55,"b":72}');
 } {1}
-do_execsql_test json-6.3 {
-  SELECT json_valid('["a",55,"b",72,]');
+do_execsql_test json-6.8 {
+  SELECT json_error_position('["a",55,"b",72,]');
 } {0}
-do_execsql_test json-6.4 {
+do_execsql_test json-6.9 {
+  SELECT json_error_position('["a",55,"b",72 , ]');
+} {0}
+do_execsql_test json-6.10 {
+  SELECT json_error_position('["a",55,"b",72,,]');
+} {16}
+do_execsql_test json-6.11 {
   SELECT json_valid('["a",55,"b",72]');
 } {1}
 
@@ -897,4 +918,97 @@ do_execsql_test json-20.3 {
   SELECT json_object('a',2e370,'b',-3e380)->>'b';
 } {-Inf}
 
+# 2023-05-02 https://sqlite.org/forum/forumpost/06c6334412
+# JSON functions should normally return NULL when given
+# a NULL value as the JSON input.
+#
+db null NULL
+do_execsql_test json-21.1 {
+  SELECT json_valid(NULL);
+} NULL
+do_execsql_test json-21.2 {
+  SELECT json_error_position(NULL);
+} NULL
+do_execsql_test json-21.3 {
+  SELECT json(NULL);
+} NULL
+do_execsql_test json-21.4 {
+  SELECT json_array(NULL);
+} {[null]}
+do_execsql_test json-21.5 {
+  SELECT json_extract(NULL);
+} NULL
+do_execsql_test json-21.6 {
+  SELECT json_insert(NULL,'$',123);
+} NULL
+do_execsql_test json-21.7 {
+  SELECT NULL->0;
+} NULL
+do_execsql_test json-21.8 {
+  SELECT NULL->>0;
+} NULL
+do_execsql_test json-21.9 {
+  SELECT '{a:5}'->NULL;
+} NULL
+do_execsql_test json-21.10 {
+  SELECT '{a:5}'->>NULL;
+} NULL
+do_catchsql_test json-21.11 {
+  SELECT json_object(NULL,5);
+} {1 {json_object() labels must be TEXT}}
+do_execsql_test json-21.12 {
+  SELECT json_patch(NULL,'{a:5}');
+} NULL
+do_execsql_test json-21.13 {
+  SELECT json_patch('{a:5}',NULL);
+} NULL
+do_execsql_test json-21.14 {
+  SELECT json_patch(NULL,NULL);
+} NULL
+do_execsql_test json-21.15 {
+  SELECT json_remove(NULL,'$');
+} NULL
+do_execsql_test json-21.16 {
+  SELECT json_remove('{a:5,b:7}',NULL);
+} NULL
+do_execsql_test json-21.17 {
+  SELECT json_replace(NULL,'$.a',123);
+} NULL
+do_execsql_test json-21.18 {
+  SELECT json_replace('{a:5,b:7}',NULL,NULL);
+} {{{"a":5,"b":7}}}
+do_execsql_test json-21.19 {
+  SELECT json_set(NULL,'$.a',123);
+} NULL
+do_execsql_test json-21.20 {
+  SELECT json_set('{a:5,b:7}',NULL,NULL);
+} {{{"a":5,"b":7}}}
+do_execsql_test json-21.21 {
+  SELECT json_type(NULL);
+} NULL
+do_execsql_test json-21.22 {
+  SELECT json_type('{a:5,b:7}',NULL);
+} NULL
+do_execsql_test json-21.23 {
+  SELECT json_quote(NULL);
+} null
+do_execsql_test json-21.24 {
+  SELECT count(*) FROM json_each(NULL);
+} 0
+do_execsql_test json-21.25 {
+  SELECT count(*) FROM json_tree(NULL);
+} 0
+do_execsql_test json-21.26 {
+  WITH c(x) AS (VALUES(1),(2.0),(NULL),('three'))
+  SELECT json_group_array(x) FROM c;
+} {[1,2.0,null,"three"]}
+do_execsql_test json-21.27 {
+  WITH c(x,y) AS (VALUES('a',1),('b',2.0),('c',NULL),(NULL,'three'),('e','four'))
+  SELECT json_group_object(x,y) FROM c;
+} {{{"a":1,"b":2.0,"c":null,:"three","e":"four"}}}
+
+
+
+
+
 finish_test

test/json102.test

@@ -301,18 +301,27 @@ for {set i 0} {$i<100} {incr i} {
 # allowing them.  The following tests verify that the problem is now
 # fixed.
 #
-do_execsql_test json102-1401 { SELECT json_valid('{"x":01}') } 0
-do_execsql_test json102-1402 { SELECT json_valid('{"x":-01}') } 0
-do_execsql_test json102-1403 { SELECT json_valid('{"x":0}') } 1
-do_execsql_test json102-1404 { SELECT json_valid('{"x":-0}') } 1
-do_execsql_test json102-1405 { SELECT json_valid('{"x":0.1}') } 1
-do_execsql_test json102-1406 { SELECT json_valid('{"x":-0.1}') } 1
-do_execsql_test json102-1407 { SELECT json_valid('{"x":0.0000}') } 1
-do_execsql_test json102-1408 { SELECT json_valid('{"x":-0.0000}') } 1
-do_execsql_test json102-1409 { SELECT json_valid('{"x":01.5}') } 0
-do_execsql_test json102-1410 { SELECT json_valid('{"x":-01.5}') } 0
-do_execsql_test json102-1411 { SELECT json_valid('{"x":00}') } 0
-do_execsql_test json102-1412 { SELECT json_valid('{"x":-00}') } 0
+foreach {id j x0 x5} {
+  1401 {'{"x":01}'} 0 0
+  1402 {'{"x":-01}'} 0 0
+  1403 {'{"x":0}'} 1 1
+  1404 {'{"x":-0}'} 1 1
+  1405 {'{"x":0.1}'} 1 1
+  1406 {'{"x":-0.1}'} 1 1
+  1407 {'{"x":0.0000}'} 1 1
+  1408 {'{"x":-0.0000}'} 1 1
+  1409 {'{"x":01.5}'} 0 0
+  1410 {'{"x":-01.5}'} 0 0
+  1411 {'{"x":00}'} 0 0
+  1412 {'{"x":-00}'} 0 0
+  1413 {'{"x":+0}'} 0 1
+  1414 {'{"x":+5}'} 0 1
+  1415 {'{"x":+5.5}'} 0 1
+} {
+  do_execsql_test json102-$id "
+     SELECT json_valid($j), NOT json_error_position($j);
+  " [list $x0 $x5]
+}
 
 #------------------------------------------------------------------------
 # 2017-04-10 ticket 6c9b5514077fed34551f98e64c09a10dc2fc8e16

test/json104.test

@@ -30,6 +30,48 @@ do_execsql_test json104-100 {
        }
      }');
 } {{{"a":"z","c":{"d":"e"}}}}
+do_execsql_test json104-101 {
+  SELECT json_patch('{
+       "a": "b",
+       "c": {
+         "d": "e",
+         "f": "g"
+       }
+     }','{
+       a:"z",
+       c: {
+         f: null
+       }
+     }');
+} {{{"a":"z","c":{"d":"e"}}}}
+do_execsql_test json104-102 {
+  SELECT json_patch('{
+       a: "b",
+       c: {
+         d: "e",
+         f: "g"
+       }
+     }','{
+       "a":"z",
+       "c": {
+         "f": null
+       }
+     }');
+} {{{"a":"z","c":{"d":"e"}}}}
+do_execsql_test json104-103 {
+  SELECT json_patch('{
+       a: "b",
+       c: {
+         d: "e",
+         f: "g"
+       }
+     }','{
+       a:"z",
+       c: {
+         f: null
+       }
+     }');
+} {{{"a":"z","c":{"d":"e"}}}}
 
 
 # This is the example from pages 4 and 5 of RFC-7396 

test/json501.test

@@ -0,0 +1,304 @@
+# 2023-04-27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements tests for the JSON5 enhancements to the
+# JSON SQL functions extension to the SQLite library.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix json501
+
+# From https://spec.json5.org/#introduction
+#
+#-----------------------------------------------------------------------------
+# Summary of Features
+# 
+# The following ECMAScript 5.1 features, which are not supported in JSON, have
+# been extended to JSON5. 
+#
+# Objects
+#
+#    1) Object keys may be an ECMAScript 5.1 IdentifierName.
+#    2) Objects may have a single trailing comma.
+#
+# Arrays
+#
+#    3) Arrays may have a single trailing comma.
+#
+# Strings
+#
+#    4) Strings may be single quoted.
+#    5) Strings may span multiple lines by escaping new line characters.
+#    6) Strings may include character escapes.
+#
+# Numbers
+#
+#    7) Numbers may be hexadecimal.
+#    8) Numbers may have a leading or trailing decimal point.
+#    9) Numbers may be IEEE 754 positive infinity, negative infinity, and NaN.
+#   10) Numbers may begin with an explicit plus sign.
+#
+# Comments
+#
+#   11) Single and multi-line comments are allowed.
+#
+# White Space
+#
+#   12) Additional white space characters are allowed.
+#-----------------------------------------------------------------------------
+#
+# Test number in this file are of the form X.Y where X is one of the item
+# numbers in the feature list above and Y is the test sequence number.
+#
+
+###############################################################################
+#    1) Object keys may be an ECMAScript 5.1 IdentifierName.
+do_execsql_test 1.1 {
+  WITH c(x) AS (VALUES('{a:5,b:6}'))
+  SELECT x->>'a', json(x), json_valid(x), NOT json_error_position(x) FROM c;
+} {5 {{"a":5,"b":6}} 0 1}
+do_execsql_test 1.2 {
+  SELECT '[7,null,{a:5,b:6},[8,9]]'->>'$[2].b';
+} {6}
+do_execsql_test 1.3 {
+  SELECT '{ $123 : 789 }'->>'$."$123"';
+} 789
+do_execsql_test 1.4 {
+  SELECT '{ _123$xyz : 789 }'->>'$."_123$xyz"';
+} 789
+do_execsql_test 1.5 {
+  SELECT '{ MNO_123$xyz : 789 }'->>'$."MNO_123$xyz"';
+} 789
+
+do_execsql_test 1.6 {
+  SELECT json('{ MNO_123$xyz : 789 }');
+} [list {{"MNO_123$xyz":789}}]
+
+do_catchsql_test 1.10 {
+  SELECT json('{ MNO_123/xyz : 789 }');
+} {1 {malformed JSON}}
+
+do_execsql_test 1.11 {
+  SELECT '{ MNO_123æxyz : 789 }'->>'MNO_123æxyz';
+} {789}
+
+###############################################################################
+#    2) Objects may have a single trailing comma.
+
+do_execsql_test 2.1 {
+  WITH c(x) AS (VALUES('{"a":5, "b":6, }'))
+  SELECT x->>'b', json(x), json_valid(x), NOT json_error_position(x) FROM c;
+} {6 {{"a":5,"b":6}} 0 1}
+do_execsql_test 2.2 {
+  SELECT '{a:5, b:6 , }'->>'b';
+} 6
+do_catchsql_test 2.3 {
+  SELECT '{a:5, b:6 ,, }'->>'b';
+} {1 {malformed JSON}}
+do_catchsql_test 2.4 {
+  SELECT '{a:5, b:6, ,}'->>'b';
+} {1 {malformed JSON}}
+
+###############################################################################
+#    3) Arrays may have a single trailing comma.
+
+do_execsql_test 3.1 {
+  WITH c(x) AS (VALUES('[5, 6,]'))
+  SELECT x->>1, json(x), json_valid(x), NOT json_error_position(x) FROM c;
+} {6 {[5,6]} 0 1}
+do_execsql_test 3.2 {
+  SELECT '[5, 6 , ]'->>1;
+} 6
+do_catchsql_test 3.3 {
+  SELECT '[5, 6,,]'->>1;
+} {1 {malformed JSON}}
+do_catchsql_test 3.4 {
+  SELECT '[5, 6 , , ]'->>1;
+} {1 {malformed JSON}}
+
+###############################################################################
+#    4) Strings may be single quoted.
+
+do_execsql_test 4.1 {
+  WITH c(x) AS (VALUES('{"a": ''abcd''}'))
+  SELECT x->>'a', json(x), json_valid(x), NOT json_error_position(x) FROM c;
+} {abcd {{"a":"abcd"}} 0 1}
+do_execsql_test 4.2 {
+  SELECT '{b: 123, ''a'': ''ab\''cd''}'->>'a';
+} {ab'cd}
+
+###############################################################################
+#    5) Strings may span multiple lines by escaping new line characters.
+
+do_execsql_test 5.1 {
+  WITH c(x) AS (VALUES('{a: "abc'||char(0x5c,0x0a)||'xyz"}'))
+  SELECT x->>'a', json(x), json_valid(x), NOT json_error_position(x) FROM c;
+} {abcxyz {{"a":"abcxyz"}} 0 1}
+do_execsql_test 5.2 {
+  SELECT ('{a: "abc'||char(0x5c,0x0d)||'xyz"}')->>'a';
+} {abcxyz}
+do_execsql_test 5.3 {
+  SELECT ('{a: "abc'||char(0x5c,0x0d,0x0a)||'xyz"}')->>'a';
+} {abcxyz}
+do_execsql_test 5.4 {
+  SELECT ('{a: "abc'||char(0x5c,0x2028)||'xyz"}')->>'a';
+} {abcxyz}
+do_execsql_test 5.5 {
+  SELECT ('{a: "abc'||char(0x5c,0x2029)||'xyz"}')->>'a';
+} {abcxyz}
+
+
+###############################################################################
+#    6) Strings may include character escapes.
+
+do_execsql_test 6.1 {
+  SELECT ('{a: "abc'||char(0x5c,0x27)||'xyz"}')->>'a';
+} {abc'xyz}
+do_execsql_test 6.2 {
+  SELECT ('{a: "abc'||char(0x5c,0x22)||'xyz"}')->>'a';
+} {abc"xyz}
+do_execsql_test 6.3 {
+  SELECT ('{a: "abc'||char(0x5c,0x5c)||'xyz"}')->>'a';
+} {{abc\xyz}}
+do_execsql_test 6.4 {
+  SELECT hex(('{a: "abc\bxyz"}')->>'a');
+} {6162630878797A}
+do_execsql_test 6.5 {
+  SELECT hex(('{a: "abc\f\n\r\t\vxyz"}')->>'a');
+} {6162630C0A0D090B78797A}
+do_execsql_test 6.6 {
+  SELECT hex(('{a: "abc\0xyz"}')->>'a');
+} {6162630078797A}
+do_execsql_test 6.7 {
+  SELECT '{a: "abc\x35\x4f\x6Exyz"}'->>'a';
+} {abc5Onxyz}
+do_execsql_test 6.8 {
+  SELECT '{a: "\x6a\x6A\x6b\x6B\x6c\x6C\x6d\x6D\x6e\x6E\x6f\x6F"}'->>'a';
+} {jjkkllmmnnoo}
+
+###############################################################################
+#    7) Numbers may be hexadecimal.
+
+do_execsql_test 7.1 {
+  SELECT '{a: 0x0}'->>'a';
+} 0
+do_execsql_test 7.2 {
+  SELECT '{a: -0x0}'->>'a';
+} 0
+do_execsql_test 7.3 {
+  SELECT '{a: +0x0}'->>'a';
+} 0
+do_execsql_test 7.4 {
+  SELECT '{a: 0xabcdef}'->>'a';
+} 11259375
+do_execsql_test 7.5 {
+  SELECT '{a: -0xaBcDeF}'->>'a';
+} -11259375
+do_execsql_test 7.6 {
+  SELECT '{a: +0xABCDEF}'->>'a';
+} 11259375
+
+###############################################################################
+#    8) Numbers may have a leading or trailing decimal point.
+
+do_execsql_test 8.1 {
+  WITH c(x) AS (VALUES('{x: 4.}')) SELECT x->>'x', json(x) FROM c;
+} {4.0 {{"x":4.0}}}
+do_execsql_test 8.2 {
+  WITH c(x) AS (VALUES('{x: +4.}')) SELECT x->>'x', json(x) FROM c;
+} {4.0 {{"x":4.0}}}
+do_execsql_test 8.3 {
+  WITH c(x) AS (VALUES('{x: -4.}')) SELECT x->>'x', json(x) FROM c;
+} {-4.0 {{"x":-4.0}}}
+do_execsql_test 8.3 {
+  WITH c(x) AS (VALUES('{x: .5}')) SELECT x->>'x', json(x) FROM c;
+} {0.5 {{"x":0.5}}}
+do_execsql_test 8.4 {
+  WITH c(x) AS (VALUES('{x: -.5}')) SELECT x->>'x', json(x) FROM c;
+} {-0.5 {{"x":-0.5}}}
+do_execsql_test 8.5 {
+  WITH c(x) AS (VALUES('{x: +.5}')) SELECT x->>'x', json(x) FROM c;
+} {0.5 {{"x":0.5}}}
+do_execsql_test 8.6 {
+  WITH c(x) AS (VALUES('{x: 4.e0}')) SELECT x->>'x', json(x) FROM c;
+} {4.0 {{"x":4.0e0}}}
+do_execsql_test 8.7 {
+  WITH c(x) AS (VALUES('{x: +4.e1}')) SELECT x->>'x', json(x) FROM c;
+} {40.0 {{"x":4.0e1}}}
+do_execsql_test 8.8 {
+  WITH c(x) AS (VALUES('{x: -4.e2}')) SELECT x->>'x', json(x) FROM c;
+} {-400.0 {{"x":-4.0e2}}}
+do_execsql_test 8.9 {
+  WITH c(x) AS (VALUES('{x: .5e3}')) SELECT x->>'x', json(x) FROM c;
+} {500.0 {{"x":0.5e3}}}
+do_execsql_test 8.10 {
+  WITH c(x) AS (VALUES('{x: -.5e-1}')) SELECT x->>'x', json(x) FROM c;
+} {-0.05 {{"x":-0.5e-1}}}
+do_execsql_test 8.11 {
+  WITH c(x) AS (VALUES('{x: +.5e-2}')) SELECT x->>'x', json(x) FROM c;
+} {0.005 {{"x":0.5e-2}}}
+
+
+###############################################################################
+#    9) Numbers may be IEEE 754 positive infinity, negative infinity, and NaN.
+
+do_execsql_test 9.1 {
+  WITH c(x) AS (VALUES('{x: +Infinity}')) SELECT x->>'x', json(x) FROM c;
+} {Inf {{"x":9.0e999}}}
+do_execsql_test 9.2 {
+  WITH c(x) AS (VALUES('{x: -Infinity}')) SELECT x->>'x', json(x) FROM c;
+} {-Inf {{"x":-9.0e999}}}
+do_execsql_test 9.3 {
+  WITH c(x) AS (VALUES('{x: Infinity}')) SELECT x->>'x', json(x) FROM c;
+} {Inf {{"x":9.0e999}}}
+do_execsql_test 9.4 {
+  WITH c(x) AS (VALUES('{x: NaN}')) SELECT x->>'x', json(x) FROM c;
+} {{} {{"x":null}}}
+
+###############################################################################
+#   10) Numbers may begin with an explicit plus sign.
+
+do_execsql_test 10.1 {
+  SELECT '{a: +123}'->'a';
+} 123
+
+###############################################################################
+#   11) Single and multi-line comments are allowed.
+
+do_execsql_test 11.1 {
+  SELECT ' /* abc */ { /*def*/ aaa /* xyz */ : // to the end of line
+          123 /* xyz */ , /* 123 */ }'->>'aaa';
+} 123
+
+###############################################################################
+#   12) Additional white space characters are allowed.
+
+do_execsql_test 12.1 {
+  SELECT (char(0x09,0x0a,0x0b,0x0c,0x0d,0x20,0xa0,0x2028,0x2029)
+          || '{a: "xyz"}')->>'a';
+} xyz
+do_execsql_test 12.2 {
+  SELECT ('{a:' || char(0x09,0x0a,0x0b,0x0c,0x0d,0x20,0xa0,0x2028,0x2029)
+          || '"xyz"}')->>'a';
+} xyz
+do_execsql_test 12.3 {
+  SELECT (char(0x1680,0x2000,0x2001,0x2002,0x2003,0x2004,0x2005,
+               0x2006,0x2007,0x2008,0x2009,0x200a,0x3000,0xfeff)
+          || '{a: "xyz"}')->>'a';
+} xyz
+do_execsql_test 12.4 {
+  SELECT ('{a: ' ||char(0x1680,0x2000,0x2001,0x2002,0x2003,0x2004,0x2005,
+                        0x2006,0x2007,0x2008,0x2009,0x200a,0x3000,0xfeff)
+          || ' "xyz"}')->>'a';
+} xyz
+
+
+finish_test

test/json502.test

@@ -0,0 +1,25 @@
+# 2023-04-28
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements tests for the JSON5 enhancements to the
+# JSON SQL functions extension to the SQLite library.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix json502
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(x JSON);
+  INSERT INTO t1(x) VALUES('{a:{b:{c:"hello",},},}');
+  SELECT fullkey FROM t1, json_tree(x);
+} {{$} {$.a} {$.a.b} {$.a.b.c}}
+
+finish_test

test/misc1.test

@@ -592,6 +592,9 @@ do_test misc1-18.1 {
   set n [sqlite3_sleep 100]
   expr {$n>=100}
 } {1}
+do_test misc1-18.2 {
+  sqlite3_sleep -100
+} {0}
 
 # 2014-01-10:  In a CREATE TABLE AS, if one or more of the column names
 # are an empty string, that is still OK.

test/shell1.test

@@ -254,6 +254,11 @@ do_test shell1-2.4.2 {
   catchcmd "test.db" ".mode \"csv\""
 } {0 {}}
 
+# check that certain quoted arg escapes work
+do_test shell1-2.5.1 {
+  catchcmd ":memory:" ".print \"\\060\\077 \\x3f\\x30 \\a\\t\""
+} [list 0 "0? ?0 \a\t"]
+
 
 #----------------------------------------------------------------------------
 # Test cases shell1-3.*: Basic test that "dot" command can be called.

test/shell2.test

@@ -216,5 +216,51 @@ do_test shell2-1.4.9 {
 done
 2}}
 
+# Verify that generate_series stays sane near 64-bit range boundaries.
+# See overflow report at https://sqlite.org/forum/forumpost/5d34ce5280
+do_test shell2-1.4.10 {
+ set res [catchcmd :memory: [string trim {
+ SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1);
+ SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1);
+ SELECT avg(rowid),min(value),max(value) FROM generate_series(
+  -9223372036854775808,9223372036854775807,1085102592571150095);
+ SELECT * FROM generate_series(-9223372036854775808,9223372036854775807,
+  9223372036854775807);
+ SELECT value,rowid FROM generate_series(-4611686018427387904,
+  4611686018427387904, 4611686018427387904) ORDER BY value DESC;
+ SELECT * FROM generate_series(0,-2,-1);
+ SELECT * FROM generate_series(0,-2);
+ SELECT * FROM generate_series(0,2) LIMIT 3;}]]
+} {0 {9223372036854775807
+9223372036854775807
+9.5|-9223372036854775808|9223372036854775807
+-9223372036854775808
+-1
+9223372036854775806
+4611686018427387904|3
+0|2
+-4611686018427387904|1
+0
+-1
+-2
+0
+1
+2}}
+
+# Bug discovered while messing around, .import hangs with
+# bit 7 set in column separator.
+do_test shell2-1.4.11 {
+  forcedelete dummy.csv
+  set df [open dummy.csv w]
+  puts $df dog,cat
+  close $df
+  set res [catchcmd :memory: [string trim {
+ CREATE TABLE t(line text);
+.mode ascii
+.separator "\377" "\n"
+.import dummy.csv t
+ SELECT count(*) FROM t;}]]
+} {0 1}
+
 
 finish_test

test/speedtest1.c

@@ -12,6 +12,7 @@ static const char zHelp[] =
   "  --checkpoint        Run PRAGMA wal_checkpoint after each test case\n"
   "  --exclusive         Enable locking_mode=EXCLUSIVE\n"
   "  --explain           Like --sqlonly but with added EXPLAIN keywords\n"
+  "  --fullfsync         Enable fullfsync=TRUE\n"
   "  --heap SZ MIN       Memory allocator uses SZ bytes & min allocation MIN\n"
   "  --incrvacuum        Enable incremenatal vacuum mode\n"
   "  --journal M         Set the journal_mode to M\n"
@@ -2203,6 +2204,7 @@ int main(int argc, char **argv){
   int doAutovac = 0;            /* True for --autovacuum */
   int cacheSize = 0;            /* Desired cache size.  0 means default */
   int doExclusive = 0;          /* True for --exclusive */
+  int doFullFSync = 0;          /* True for --fullfsync */
   int nHeap = 0, mnHeap = 0;    /* Heap size from --heap */
   int doIncrvac = 0;            /* True for --incrvacuum */
   const char *zJMode = 0;       /* Journal mode */
@@ -2267,6 +2269,8 @@ int main(int argc, char **argv){
         cacheSize = integerValue(argv[++i]);
       }else if( strcmp(z,"exclusive")==0 ){
         doExclusive = 1;
+      }else if( strcmp(z,"fullfsync")==0 ){
+        doFullFSync = 1;
       }else if( strcmp(z,"checkpoint")==0 ){
         g.doCheckpoint = 1;
       }else if( strcmp(z,"explain")==0 ){
@@ -2511,7 +2515,11 @@ int main(int argc, char **argv){
   if( cacheSize ){
     speedtest1_exec("PRAGMA cache_size=%d", cacheSize);
   }
-  if( noSync ) speedtest1_exec("PRAGMA synchronous=OFF");
+  if( noSync ){
+    speedtest1_exec("PRAGMA synchronous=OFF");
+  }else if( doFullFSync ){
+    speedtest1_exec("PRAGMA fullfsync=ON");
+  }
   if( doExclusive ){
     speedtest1_exec("PRAGMA locking_mode=EXCLUSIVE");
   }

test/tabfunc01.test

@@ -61,7 +61,7 @@ do_execsql_test tabfunc01-1.8 {
 } {30 25 20 15 10 5 0}
 do_execsql_test tabfunc01-1.9 {
   SELECT rowid, * FROM generate_series(0,32,5) ORDER BY value DESC;
-} {1 30 2 25 3 20 4 15 5 10 6 5 7 0}
+} {7 30 6 25 5 20 4 15 3 10 2 5 1 0}
 do_execsql_test tabfunc01-1.10 {
   SELECT rowid, * FROM generate_series(0,32,5) ORDER BY +value DESC;
 } {7 30 6 25 5 20 4 15 3 10 2 5 1 0}

test/tkt-99378177930f87bd.test

@@ -178,6 +178,8 @@ do_execsql_test tkt-99378-310 {
 
 # 2023-03-04 https://sqlite.org/forum/forumpost/a68313d054
 #
+# See also indexexpr1-2200 added on 2023-03-18.
+#
 do_execsql_test tkt-99378-400 {
   DROP TABLE t1;
   CREATE TABLE t0(w);