Import all FTS3/4 enhancements and fixes that exist on the latest trunk

(3.31.0-beta) that do not require extensive change to the SQLite core into the 3.22 branch. Basically, the 3.31.0 FTS3 sources are copied into 3.22.0, with minor changes to work around core enhancements that are not available in 3.22.0. FossilOrigin-Name: cbcbb1e5a98bde5a40e90faf8467b4f761d2ad7414cf4423571bdb8ba38a81d5
Fix a buffer overread that could occur when running fts5 prefix queries inside a transaction.
2020-01-17 14:18:48 +00:00 · 2019-09-03 18:36:11 +00:00 · 2019-09-03 18:04:15 +00:00 · 2018-12-26 14:23:14 +00:00 · 2018-12-26 11:39:39 +00:00 · 2018-12-24 13:39:13 +00:00
29 changed files with 1412 additions and 478 deletions
@@ -52,8 +52,10 @@

  SECURITY: If the fts3 extension is used in an environment where potentially
    malicious users may execute arbitrary SQL (i.e. gears), they should be
-    prevented from invoking the fts3_tokenizer() function, possibly using the
-    authorisation callback.
+    prevented from invoking the fts3_tokenizer() function.  The
+    fts3_tokenizer() function is disabled by default. It is only enabled
+    by SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER. Do not enable it in
+    security sensitive environments.

  See "Sample code" below for an example of calling the fts3_tokenizer()
  function from C code.
@@ -320,6 +320,14 @@ int sqlite3Fts3Never(int b)  { assert( !b ); return b; }
 # endif
 #endif

+/*
+** This variable is set to false when running tests for which the on disk
+** structures should not be corrupt. Otherwise, true. If it is false, extra
+** assert() conditions in the fts3 code are activated - conditions that are
+** only true if it is guaranteed that the fts3 database is not corrupt.
+*/
+int sqlite3_fts3_may_be_corrupt = 1;
+
 /* 
 ** Write a 64-bit variable-length integer to memory starting at p[0].
 ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
@@ -338,18 +346,13 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
 }

 #define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \
-  v = (v & mask1) | ( (*ptr++) << shift );                    \
+  v = (v & mask1) | ( (*(const unsigned char*)(ptr++)) << shift );  \
  if( (v & mask2)==0 ){ var = v; return ret; }
 #define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \
  v = (*ptr++);                                               \
  if( (v & mask2)==0 ){ var = v; return ret; }

-/* 
-** Read a 64-bit variable-length integer from memory starting at p[0].
-** Return the number of bytes read, or 0 on error.
-** The value is stored in *v.
-*/
-int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
+int sqlite3Fts3GetVarintU(const char *pBuf, sqlite_uint64 *v){
  const unsigned char *p = (const unsigned char*)pBuf;
  const unsigned char *pStart = p;
  u32 a;
@@ -371,25 +374,61 @@ int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
  return (int)(p - pStart);
 }

+/* 
+** Read a 64-bit variable-length integer from memory starting at p[0].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
+  return sqlite3Fts3GetVarintU(pBuf, (sqlite3_uint64*)v);
+}
+
+/* 
+** Read a 64-bit variable-length integer from memory starting at p[0] and
+** not extending past pEnd[-1].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+int sqlite3Fts3GetVarintBounded(
+  const char *pBuf,
+  const char *pEnd,
+  sqlite_int64 *v
+){
+  const unsigned char *p = (const unsigned char*)pBuf;
+  const unsigned char *pStart = p;
+  const unsigned char *pX = (const unsigned char*)pEnd;
+  u64 b = 0;
+  int shift;
+  for(shift=0; shift<=63; shift+=7){
+    u64 c = p<pX ? *p : 0;
+    p++;
+    b += (c&0x7F) << shift;
+    if( (c & 0x80)==0 ) break;
+  }
+  *v = b;
+  return (int)(p - pStart);
+}
+
 /*
 ** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to 
 ** a non-negative 32-bit integer before it is returned.
 */
 int sqlite3Fts3GetVarint32(const char *p, int *pi){
+  const unsigned char *ptr = (const unsigned char*)p;
  u32 a;

 #ifndef fts3GetVarint32
-  GETVARINT_INIT(a, p, 0,  0x00,     0x80, *pi, 1);
+  GETVARINT_INIT(a, ptr, 0,  0x00,     0x80, *pi, 1);
 #else
-  a = (*p++);
+  a = (*ptr++);
  assert( a & 0x80 );
 #endif

-  GETVARINT_STEP(a, p, 7,  0x7F,     0x4000, *pi, 2);
-  GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *pi, 3);
-  GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4);
+  GETVARINT_STEP(a, ptr, 7,  0x7F,     0x4000, *pi, 2);
+  GETVARINT_STEP(a, ptr, 14, 0x3FFF,   0x200000, *pi, 3);
+  GETVARINT_STEP(a, ptr, 21, 0x1FFFFF, 0x10000000, *pi, 4);
  a = (a & 0x0FFFFFFF );
-  *pi = (int)(a | ((u32)(*p & 0x07) << 28));
+  *pi = (int)(a | ((u32)(*ptr & 0x07) << 28));
  assert( 0==(a & 0x80000000) );
  assert( *pi>=0 );
  return 5;
@@ -560,13 +599,18 @@ static int fts3DestroyMethod(sqlite3_vtab *pVtab){
  sqlite3 *db = p->db;             /* Database handle */

  /* Drop the shadow tables */
-  if( p->zContentTbl==0 ){
-    fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName);
-  }
-  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName);
-  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName);
-  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName);
-  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName);
+  fts3DbExec(&rc, db, 
+    "DROP TABLE IF EXISTS %Q.'%q_segments';"
+    "DROP TABLE IF EXISTS %Q.'%q_segdir';"
+    "DROP TABLE IF EXISTS %Q.'%q_docsize';"
+    "DROP TABLE IF EXISTS %Q.'%q_stat';"
+    "%s DROP TABLE IF EXISTS %Q.'%q_content';",
+    zDb, p->zName,
+    zDb, p->zName,
+    zDb, p->zName,
+    zDb, p->zName,
+    (p->zContentTbl ? "--" : ""), zDb,p->zName
+  );

  /* If everything has worked, invoke fts3DisconnectMethod() to free the
  ** memory associated with the Fts3Table structure and return SQLITE_OK.
@@ -798,10 +842,10 @@ static void fts3Appendf(
 ** memory.
 */
 static char *fts3QuoteId(char const *zInput){
-  int nRet;
+  sqlite3_int64 nRet;
  char *zRet;
  nRet = 2 + (int)strlen(zInput)*2 + 1;
-  zRet = sqlite3_malloc(nRet);
+  zRet = sqlite3_malloc64(nRet);
  if( zRet ){
    int i;
    char *z = zRet;
@@ -982,7 +1026,7 @@ static int fts3PrefixParameter(
    }
  }

-  aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
+  aIndex = sqlite3_malloc64(sizeof(struct Fts3Index) * nIndex);
  *apIndex = aIndex;
  if( !aIndex ){
    return SQLITE_NOMEM;
@@ -1061,7 +1105,7 @@ static int fts3ContentColumns(

  if( rc==SQLITE_OK ){
    const char **azCol;           /* Output array */
-    int nStr = 0;                 /* Size of all column names (incl. 0x00) */
+    sqlite3_int64 nStr = 0;       /* Size of all column names (incl. 0x00) */
    int nCol;                     /* Number of table columns */
    int i;                        /* Used to iterate through columns */

@@ -1071,11 +1115,11 @@ static int fts3ContentColumns(
    nCol = sqlite3_column_count(pStmt);
    for(i=0; i<nCol; i++){
      const char *zCol = sqlite3_column_name(pStmt, i);
-      nStr += (int)strlen(zCol) + 1;
+      nStr += strlen(zCol) + 1;
    }

    /* Allocate and populate the array to return. */
-    azCol = (const char **)sqlite3_malloc(sizeof(char *) * nCol + nStr);
+    azCol = (const char **)sqlite3_malloc64(sizeof(char *) * nCol + nStr);
    if( azCol==0 ){
      rc = SQLITE_NOMEM;
    }else{
@@ -1123,7 +1167,7 @@ static int fts3InitVtab(
  Fts3Table *p = 0;               /* Pointer to allocated vtab */
  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* Iterator variable */
-  int nByte;                      /* Size of allocation used for *p */
+  sqlite3_int64 nByte;            /* Size of allocation used for *p */
  int iCol;                       /* Column index */
  int nString = 0;                /* Bytes required to hold all column names */
  int nCol = 0;                   /* Number of columns in the FTS table */
@@ -1157,10 +1201,10 @@ static int fts3InitVtab(
  nName = (int)strlen(argv[2]) + 1;

  nByte = sizeof(const char *) * (argc-2);
-  aCol = (const char **)sqlite3_malloc(nByte);
+  aCol = (const char **)sqlite3_malloc64(nByte);
  if( aCol ){
    memset((void*)aCol, 0, nByte);
-    azNotindexed = (char **)sqlite3_malloc(nByte);
+    azNotindexed = (char **)sqlite3_malloc64(nByte);
  }
  if( azNotindexed ){
    memset(azNotindexed, 0, nByte);
@@ -1355,7 +1399,7 @@ static int fts3InitVtab(
          nName +                              /* zName */
          nDb +                                /* zDb */
          nString;                             /* Space for azColumn strings */
-  p = (Fts3Table*)sqlite3_malloc(nByte);
+  p = (Fts3Table*)sqlite3_malloc64(nByte);
  if( p==0 ){
    rc = SQLITE_NOMEM;
    goto fts3_init_out;
@@ -1460,6 +1504,10 @@ static int fts3InitVtab(
  fts3DatabasePageSize(&rc, p);
  p->nNodeSize = p->nPgsz-35;

+#if defined(SQLITE_DEBUG)||defined(SQLITE_TEST)
+  p->nMergeCount = FTS3_MERGE_COUNT;
+#endif
+
  /* Declare the table schema to SQLite. */
  fts3DeclareVtab(&rc, p);

@@ -1555,6 +1603,10 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  int iDocidLe = -1;              /* Index of docid<=x constraint, if present */
  int iIdx;

+  if( p->bLock ){
+    return SQLITE_ERROR;
+  }
+
  /* By default use a full table scan. This is an expensive option,
  ** so search through the constraints to see if a more efficient 
  ** strategy is possible.
@@ -1753,7 +1805,11 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr){
    }else{
      zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
      if( !zSql ) return SQLITE_NOMEM;
-      rc = sqlite3_prepare_v3(p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
+      p->bLock++;
+      rc = sqlite3_prepare_v3(
+          p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0
+      );
+      p->bLock--;
      sqlite3_free(zSql);
    }
    if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1;
@@ -1771,11 +1827,15 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
  if( pCsr->isRequireSeek ){
    rc = fts3CursorSeekStmt(pCsr);
    if( rc==SQLITE_OK ){
+      Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab;
+      pTab->bLock++;
      sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
      pCsr->isRequireSeek = 0;
      if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
+        pTab->bLock--;
        return SQLITE_OK;
      }else{
+        pTab->bLock--;
        rc = sqlite3_reset(pCsr->pStmt);
        if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){
          /* If no row was found and no error has occurred, then the %_content
@@ -1821,7 +1881,7 @@ static int fts3ScanInteriorNode(
  const char *zCsr = zNode;       /* Cursor to iterate through node */
  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
  char *zBuffer = 0;              /* Buffer to load terms into */
-  int nAlloc = 0;                 /* Size of allocated buffer */
+  i64 nAlloc = 0;                 /* Size of allocated buffer */
  int isFirstTerm = 1;            /* True when processing first term on page */
  sqlite3_int64 iChild;           /* Block id of child node to descend to */

@@ -1859,14 +1919,14 @@ static int fts3ScanInteriorNode(
    zCsr += fts3GetVarint32(zCsr, &nSuffix);
    
    assert( nPrefix>=0 && nSuffix>=0 );
-    if( &zCsr[nSuffix]>zEnd ){
+    if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr || nSuffix==0 ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
-    if( nPrefix+nSuffix>nAlloc ){
+    if( (i64)nPrefix+nSuffix>nAlloc ){
      char *zNew;
-      nAlloc = (nPrefix+nSuffix) * 2;
-      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+      nAlloc = ((i64)nPrefix+nSuffix) * 2;
+      zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
      if( !zNew ){
        rc = SQLITE_NOMEM;
        goto finish_scan;
@@ -1947,7 +2007,7 @@ static int fts3SelectLeaf(

  fts3GetVarint32(zNode, &iHeight);
  rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
-  assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
+  assert_fts3_nc( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );

  if( rc==SQLITE_OK && iHeight>1 ){
    char *zBlob = 0;              /* Blob read from %_segments table */
@@ -1967,7 +2027,13 @@ static int fts3SelectLeaf(
      rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0);
    }
    if( rc==SQLITE_OK ){
-      rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
+      int iNewHeight = 0;
+      fts3GetVarint32(zBlob, &iNewHeight);
+      if( iNewHeight>=iHeight ){
+        rc = FTS_CORRUPT_VTAB;
+      }else{
+        rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
+      }
    }
    sqlite3_free(zBlob);
  }
@@ -2072,10 +2138,11 @@ static void fts3ColumnlistCopy(char **pp, char **ppPoslist){
 }

 /*
-** Value used to signify the end of an position-list. This is safe because
-** it is not possible to have a document with 2^31 terms.
+** Value used to signify the end of an position-list. This must be
+** as large or larger than any value that might appear on the
+** position-list, even a position list that has been corrupted.
 */
-#define POSITION_LIST_END 0x7fffffff
+#define POSITION_LIST_END LARGEST_INT64

 /*
 ** This function is used to help parse position-lists. When this function is
@@ -2134,7 +2201,7 @@ static int fts3PutColNumber(char **pp, int iCol){
 ** updated appropriately.   The caller is responsible for insuring
 ** that there is enough space in *pp to hold the complete output.
 */
-static void fts3PoslistMerge(
+static int fts3PoslistMerge(
  char **pp,                      /* Output buffer */
  char **pp1,                     /* Left input list */
  char **pp2                      /* Right input list */
@@ -2147,12 +2214,18 @@ static void fts3PoslistMerge(
    int iCol1;         /* The current column index in pp1 */
    int iCol2;         /* The current column index in pp2 */

-    if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1);
-    else if( *p1==POS_END ) iCol1 = POSITION_LIST_END;
+    if( *p1==POS_COLUMN ){ 
+      fts3GetVarint32(&p1[1], &iCol1);
+      if( iCol1==0 ) return FTS_CORRUPT_VTAB;
+    }
+    else if( *p1==POS_END ) iCol1 = 0x7fffffff;
    else iCol1 = 0;

-    if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2);
-    else if( *p2==POS_END ) iCol2 = POSITION_LIST_END;
+    if( *p2==POS_COLUMN ){
+      fts3GetVarint32(&p2[1], &iCol2);
+      if( iCol2==0 ) return FTS_CORRUPT_VTAB;
+    }
+    else if( *p2==POS_END ) iCol2 = 0x7fffffff;
    else iCol2 = 0;

    if( iCol1==iCol2 ){
@@ -2199,6 +2272,7 @@ static void fts3PoslistMerge(
  *pp = p;
  *pp1 = p1 + 1;
  *pp2 = p2 + 1;
+  return SQLITE_OK;
 }

 /*
@@ -2263,10 +2337,9 @@ static int fts3PoslistPhraseMerge(
        p += sqlite3Fts3PutVarint(p, iCol1);
      }

-      assert( *p1!=POS_END && *p1!=POS_COLUMN );
-      assert( *p2!=POS_END && *p2!=POS_COLUMN );
      fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
      fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
+      if( iPos1<0 || iPos2<0 ) break;

      while( 1 ){
        if( iPos2==iPos1+nToken 
@@ -2415,12 +2488,12 @@ static void fts3GetDeltaVarint3(
  if( *pp>=pEnd ){
    *pp = 0;
  }else{
-    sqlite3_int64 iVal;
-    *pp += sqlite3Fts3GetVarint(*pp, &iVal);
+    u64 iVal;
+    *pp += sqlite3Fts3GetVarintU(*pp, &iVal);
    if( bDescIdx ){
-      *pVal -= iVal;
+      *pVal = (i64)((u64)*pVal - iVal);
    }else{
-      *pVal += iVal;
+      *pVal = (i64)((u64)*pVal + iVal);
    }
  }
 }
@@ -2447,14 +2520,16 @@ static void fts3PutDeltaVarint3(
  int *pbFirst,                   /* IN/OUT: True after first int written */
  sqlite3_int64 iVal              /* Write this value to the list */
 ){
-  sqlite3_int64 iWrite;
+  sqlite3_uint64 iWrite;
  if( bDescIdx==0 || *pbFirst==0 ){
-    iWrite = iVal - *piPrev;
+    assert_fts3_nc( *pbFirst==0 || iVal>=*piPrev );
+    iWrite = (u64)iVal - (u64)*piPrev;
  }else{
-    iWrite = *piPrev - iVal;
+    assert_fts3_nc( *piPrev>=iVal );
+    iWrite = (u64)*piPrev - (u64)iVal;
  }
  assert( *pbFirst || *piPrev==0 );
-  assert( *pbFirst==0 || iWrite>0 );
+  assert_fts3_nc( *pbFirst==0 || iWrite>0 );
  *pp += sqlite3Fts3PutVarint(*pp, iWrite);
  *piPrev = iVal;
  *pbFirst = 1;
@@ -2470,7 +2545,8 @@ static void fts3PutDeltaVarint3(
 ** Using this makes it easier to write code that can merge doclists that are
 ** sorted in either ascending or descending order.
 */
-#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2))
+/* #define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i64)((u64)i1-i2)) */
+#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1>i2?1:((i1==i2)?0:-1)))

 /*
 ** This function does an "OR" merge of two doclists (output contains all
@@ -2492,6 +2568,7 @@ static int fts3DoclistOrMerge(
  char *a2, int n2,               /* Second doclist */
  char **paOut, int *pnOut        /* OUT: Malloc'd doclist */
 ){
+  int rc = SQLITE_OK;
  sqlite3_int64 i1 = 0;
  sqlite3_int64 i2 = 0;
  sqlite3_int64 iPrev = 0;
@@ -2535,7 +2612,7 @@ static int fts3DoclistOrMerge(
  ** A symetric argument may be made if the doclists are in descending 
  ** order.
  */
-  aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1);
+  aOut = sqlite3_malloc64((i64)n1+n2+FTS3_VARINT_MAX-1+FTS3_BUFFER_PADDING);
  if( !aOut ) return SQLITE_NOMEM;

  p = aOut;
@@ -2546,7 +2623,8 @@ static int fts3DoclistOrMerge(

    if( p2 && p1 && iDiff==0 ){
      fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
-      fts3PoslistMerge(&p, &p1, &p2);
+      rc = fts3PoslistMerge(&p, &p1, &p2);
+      if( rc ) break;
      fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
      fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
    }else if( !p2 || (p1 && iDiff<0) ){
@@ -2558,12 +2636,20 @@ static int fts3DoclistOrMerge(
      fts3PoslistCopy(&p, &p2);
      fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
    }
+    
+    assert( (p-aOut)<=((p1?(p1-a1):n1)+(p2?(p2-a2):n2)+FTS3_VARINT_MAX-1) );
  }

+  if( rc!=SQLITE_OK ){
+    sqlite3_free(aOut);
+    p = aOut = 0;
+  }else{
+    assert( (p-aOut)<=n1+n2+FTS3_VARINT_MAX-1 );
+    memset(&aOut[(p-aOut)], 0, FTS3_BUFFER_PADDING);
+  }
  *paOut = aOut;
  *pnOut = (int)(p-aOut);
-  assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 );
-  return SQLITE_OK;
+  return rc;
 }

 /*
@@ -2598,7 +2684,7 @@ static int fts3DoclistPhraseMerge(

  assert( nDist>0 );
  if( bDescDoclist ){
-    aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX);
+    aOut = sqlite3_malloc64((sqlite3_int64)*pnRight + FTS3_VARINT_MAX);
    if( aOut==0 ) return SQLITE_NOMEM;
  }else{
    aOut = aRight;
@@ -2782,6 +2868,7 @@ static int fts3TermSelectMerge(
    pTS->anOutput[0] = nDoclist;
    if( pTS->aaOutput[0] ){
      memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
+      memset(&pTS->aaOutput[0][nDoclist], 0, FTS3_VARINT_MAX);
    }else{
      return SQLITE_NOMEM;
    }
@@ -2833,8 +2920,8 @@ static int fts3SegReaderCursorAppend(
 ){
  if( (pCsr->nSegment%16)==0 ){
    Fts3SegReader **apNew;
-    int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
-    apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
+    sqlite3_int64 nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
+    apNew = (Fts3SegReader **)sqlite3_realloc64(pCsr->apSegment, nByte);
    if( !apNew ){
      sqlite3Fts3SegReaderFree(pNew);
      return SQLITE_NOMEM;
@@ -2873,7 +2960,7 @@ static int fts3SegReaderCursor(
  ** Fts3SegReaderPending might segfault, as the data structures used by 
  ** fts4aux are not completely populated. So it's easiest to filter these
  ** calls out here.  */
-  if( iLevel<0 && p->aIndex ){
+  if( iLevel<0 && p->aIndex && p->iPrevLangid==iLangid ){
    Fts3SegReader *pSeg = 0;
    rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg);
    if( rc==SQLITE_OK && pSeg ){
@@ -2898,7 +2985,7 @@ static int fts3SegReaderCursor(

      /* If zTerm is not NULL, and this segment is not stored entirely on its
      ** root node, the range of leaves scanned can be reduced. Do this. */
-      if( iStartBlock && zTerm ){
+      if( iStartBlock && zTerm && zRoot ){
        sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
        rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
        if( rc!=SQLITE_OK ) goto finished;
@@ -3136,6 +3223,8 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
  int rc;
  Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){
+    Fts3Table *pTab = (Fts3Table*)pCursor->pVtab;
+    pTab->bLock++;
    if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
      pCsr->isEof = 1;
      rc = sqlite3_reset(pCsr->pStmt);
@@ -3143,6 +3232,7 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
      pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
      rc = SQLITE_OK;
    }
+    pTab->bLock--;
  }else{
    rc = fts3EvalNext((Fts3Cursor *)pCursor);
  }
@@ -3150,18 +3240,6 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
  return rc;
 }

-/*
-** The following are copied from sqliteInt.h.
-**
-** Constants for the largest and smallest possible 64-bit signed integers.
-** These macros are designed to work correctly on both 32-bit and 64-bit
-** compilers.
-*/
-#ifndef SQLITE_AMALGAMATION
-# define LARGEST_INT64  (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32))
-# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64)
-#endif
-
 /*
 ** If the numeric type of argument pVal is "integer", then return it
 ** converted to a 64-bit signed integer. Otherwise, return a copy of
@@ -3215,6 +3293,10 @@ static int fts3FilterMethod(
  UNUSED_PARAMETER(idxStr);
  UNUSED_PARAMETER(nVal);

+  if( p->bLock ){
+    return SQLITE_ERROR;
+  }
+
  eSearch = (idxNum & 0x0000FFFF);
  assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
  assert( p->pSegments==0 );
@@ -3286,7 +3368,11 @@ static int fts3FilterMethod(
      );
    }
    if( zSql ){
-      rc = sqlite3_prepare_v3(p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
+      p->bLock++;
+      rc = sqlite3_prepare_v3(
+          p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0
+      );
+      p->bLock--;
      sqlite3_free(zSql);
    }else{
      rc = SQLITE_NOMEM;
@@ -3808,7 +3894,7 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  int rc = SQLITE_OK;
  UNUSED_PARAMETER(iSavepoint);
  assert( ((Fts3Table *)pVtab)->inTransaction );
-  assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
+  assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
  TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
  if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
    rc = fts3SyncMethod(pVtab);
@@ -3840,14 +3926,13 @@ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts3Table *p = (Fts3Table*)pVtab;
  UNUSED_PARAMETER(iSavepoint);
  assert( p->inTransaction );
-  assert( p->mxSavepoint >= iSavepoint );
  TESTONLY( p->mxSavepoint = iSavepoint );
  sqlite3Fts3PendingTermsClear(p);
  return SQLITE_OK;
 }

 static const sqlite3_module fts3Module = {
-  /* iVersion      */ 2,
+  /* iVersion      */ 3,
  /* xCreate       */ fts3CreateMethod,
  /* xConnect      */ fts3ConnectMethod,
  /* xBestIndex    */ fts3BestIndexMethod,
@@ -3963,7 +4048,7 @@ int sqlite3Fts3Init(sqlite3 *db){

 #ifdef SQLITE_TEST
  if( rc==SQLITE_OK ){
-    rc = sqlite3Fts3ExprInitTestInterface(db);
+    rc = sqlite3Fts3ExprInitTestInterface(db, pHash);
  }
 #endif

@@ -4150,6 +4235,7 @@ static int fts3EvalPhraseLoad(
  return rc;
 }

+#ifndef SQLITE_DISABLE_FTS4_DEFERRED
 /*
 ** This function is called on each phrase after the position lists for
 ** any deferred tokens have been loaded into memory. It updates the phrases
@@ -4253,6 +4339,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){

  return SQLITE_OK;
 }
+#endif /* SQLITE_DISABLE_FTS4_DEFERRED */

 /*
 ** Maximum number of tokens a phrase may have to be considered for the
@@ -4286,7 +4373,7 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
  int bIncrOk = (bOptOk 
   && pCsr->bDesc==pTab->bDescIdx 
   && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0
-#ifdef SQLITE_TEST
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
   && pTab->bNoIncrDoclist==0
 #endif
  );
@@ -4428,15 +4515,16 @@ static void fts3EvalDlPhraseNext(
  u8 *pbEof
 ){
  char *pIter;                            /* Used to iterate through aAll */
-  char *pEnd = &pDL->aAll[pDL->nAll];     /* 1 byte past end of aAll */
+  char *pEnd;                             /* 1 byte past end of aAll */
 
  if( pDL->pNextDocid ){
    pIter = pDL->pNextDocid;
+    assert( pDL->aAll!=0 || pIter==0 );
  }else{
    pIter = pDL->aAll;
  }

-  if( pIter>=pEnd ){
+  if( pIter==0 || pIter>=(pEnd = pDL->aAll + pDL->nAll) ){
    /* We have already reached the end of this doclist. EOF. */
    *pbEof = 1;
  }else{
@@ -4597,9 +4685,10 @@ static int fts3EvalIncrPhraseNext(
      if( bEof==0 ){
        int nList = 0;
        int nByte = a[p->nToken-1].nList;
-        char *aDoclist = sqlite3_malloc(nByte+1);
+        char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING);
        if( !aDoclist ) return SQLITE_NOMEM;
        memcpy(aDoclist, a[p->nToken-1].pList, nByte+1);
+        memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING);

        for(i=0; i<(p->nToken-1); i++){
          if( a[i].bIgnore==0 ){
@@ -4807,12 +4896,13 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
    rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
    if( rc!=SQLITE_OK ) return rc;
    a = sqlite3_column_blob(pStmt, 0);
-    assert( a );
-
-    pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
-    a += sqlite3Fts3GetVarint(a, &nDoc);
-    while( a<pEnd ){
-      a += sqlite3Fts3GetVarint(a, &nByte);
+    testcase( a==0 );  /* If %_stat.value set to X'' */
+    if( a ){
+      pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
+      a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc);
+      while( a<pEnd ){
+        a += sqlite3Fts3GetVarintBounded(a, pEnd, &nByte);
+      }
    }
    if( nDoc==0 || nByte==0 ){
      sqlite3_reset(pStmt);
@@ -4990,7 +5080,7 @@ static int fts3EvalStart(Fts3Cursor *pCsr){
  if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){
    Fts3TokenAndCost *aTC;
    Fts3Expr **apOr;
-    aTC = (Fts3TokenAndCost *)sqlite3_malloc(
+    aTC = (Fts3TokenAndCost *)sqlite3_malloc64(
        sizeof(Fts3TokenAndCost) * nToken
      + sizeof(Fts3Expr *) * nOr * 2
    );
@@ -5301,7 +5391,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
   && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
  ){
    Fts3Expr *p; 
-    int nTmp = 0;                 /* Bytes of temp space */
+    sqlite3_int64 nTmp = 0;       /* Bytes of temp space */
    char *aTmp;                   /* Temp space for PoslistNearMerge() */

    /* Allocate temporary working space. */
@@ -5310,7 +5400,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
      nTmp += p->pRight->pPhrase->doclist.nList;
    }
    nTmp += p->pPhrase->doclist.nList;
-    aTmp = sqlite3_malloc(nTmp*2);
+    aTmp = sqlite3_malloc64(nTmp*2);
    if( !aTmp ){
      *pRc = SQLITE_NOMEM;
      res = 0;
@@ -5580,15 +5670,14 @@ static void fts3EvalRestart(
 ** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase 
 ** expression nodes.
 */
-static void fts3EvalUpdateCounts(Fts3Expr *pExpr){
+static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){
  if( pExpr ){
    Fts3Phrase *pPhrase = pExpr->pPhrase;
    if( pPhrase && pPhrase->doclist.pList ){
      int iCol = 0;
      char *p = pPhrase->doclist.pList;

-      assert( *p );
-      while( 1 ){
+      do{
        u8 c = 0;
        int iCnt = 0;
        while( 0xFE & (*p | c) ){
@@ -5604,11 +5693,11 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr){
        if( *p==0x00 ) break;
        p++;
        p += fts3GetVarint32(p, &iCol);
-      }
+      }while( iCol<nCol );
    }

-    fts3EvalUpdateCounts(pExpr->pLeft);
-    fts3EvalUpdateCounts(pExpr->pRight);
+    fts3EvalUpdateCounts(pExpr->pLeft, nCol);
+    fts3EvalUpdateCounts(pExpr->pRight, nCol);
  }
 }

@@ -5652,7 +5741,7 @@ static int fts3EvalGatherStats(
    for(p=pRoot; p; p=p->pLeft){
      Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
      assert( pE->aMI==0 );
-      pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32));
+      pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32));
      if( !pE->aMI ) return SQLITE_NOMEM;
      memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
    }
@@ -5678,7 +5767,7 @@ static int fts3EvalGatherStats(
      );

      if( rc==SQLITE_OK && pCsr->isEof==0 ){
-        fts3EvalUpdateCounts(pRoot);
+        fts3EvalUpdateCounts(pRoot, pTab->nColumn);
      }
    }

@@ -95,6 +95,8 @@ SQLITE_EXTENSION_INIT3
 */
 #define FTS3_VARINT_MAX 10

+#define FTS3_BUFFER_PADDING 8
+
 /*
 ** FTS4 virtual tables may maintain multiple indexes - one index of all terms
 ** in the document set and zero or more prefix indexes. All indexes are stored
@@ -127,6 +129,18 @@ SQLITE_EXTENSION_INIT3
 #define POS_COLUMN  (1)     /* Column-list terminator */
 #define POS_END     (0)     /* Position-list terminator */ 

+/*
+** The assert_fts3_nc() macro is similar to the assert() macro, except that it
+** is used for assert() conditions that are true only if it can be 
+** guranteed that the database is not corrupt.
+*/
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
+extern int sqlite3_fts3_may_be_corrupt;
+# define assert_fts3_nc(x) assert(sqlite3_fts3_may_be_corrupt || (x))
+#else
+# define assert_fts3_nc(x) assert(x)
+#endif
+
 /*
 ** This section provides definitions to allow the
 ** FTS3 extension to be compiled outside of the 
@@ -182,6 +196,9 @@ typedef sqlite3_int64 i64;        /* 8-byte signed integer */
 # define TESTONLY(X)
 #endif

+#define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
+#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
+
 #endif /* SQLITE_AMALGAMATION */

 #ifdef SQLITE_DEBUG
@@ -225,6 +242,7 @@ struct Fts3Table {
  char *zLanguageid;              /* languageid=xxx option, or NULL */
  int nAutoincrmerge;             /* Value configured by 'automerge' */
  u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
+  int bLock;                      /* Used to prevent recursive content= tbls */

  /* Precompiled statements used by the implementation. Each of these 
  ** statements is run and reset within a single virtual table API call. 
@@ -283,13 +301,23 @@ struct Fts3Table {
  int mxSavepoint;       /* Largest valid xSavepoint integer */
 #endif

-#ifdef SQLITE_TEST
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
  /* True to disable the incremental doclist optimization. This is controled
  ** by special insert command 'test-no-incr-doclist'.  */
  int bNoIncrDoclist;
+
+  /* Number of segments in a level */
+  int nMergeCount;
 #endif
 };

+/* Macro to find the number of segments to merge */
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
+# define MergeCount(P) ((P)->nMergeCount)
+#else
+# define MergeCount(P) FTS3_MERGE_COUNT
+#endif
+
 /*
 ** When the core wants to read from the virtual table, it creates a
 ** virtual table cursor (an instance of the following structure) using
@@ -553,6 +581,8 @@ int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
 void sqlite3Fts3ErrMsg(char**,const char*,...);
 int sqlite3Fts3PutVarint(char *, sqlite3_int64);
 int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
+int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *);
+int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*);
 int sqlite3Fts3GetVarint32(const char *, int *);
 int sqlite3Fts3VarintLen(sqlite3_uint64);
 void sqlite3Fts3Dequote(char *);
@@ -584,7 +614,7 @@ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
 );
 void sqlite3Fts3ExprFree(Fts3Expr *);
 #ifdef SQLITE_TEST
-int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
 int sqlite3Fts3InitTerm(sqlite3 *db);
 #endif

@@ -66,7 +66,7 @@ static int fts3auxConnectMethod(
  char const *zFts3;              /* Name of fts3 table */
  int nDb;                        /* Result of strlen(zDb) */
  int nFts3;                      /* Result of strlen(zFts3) */
-  int nByte;                      /* Bytes of space to allocate here */
+  sqlite3_int64 nByte;            /* Bytes of space to allocate here */
  int rc;                         /* value returned by declare_vtab() */
  Fts3auxTable *p;                /* Virtual table object to return */

@@ -98,7 +98,7 @@ static int fts3auxConnectMethod(
  if( rc!=SQLITE_OK ) return rc;

  nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
-  p = (Fts3auxTable *)sqlite3_malloc(nByte);
+  p = (Fts3auxTable *)sqlite3_malloc64(nByte);
  if( !p ) return SQLITE_NOMEM;
  memset(p, 0, nByte);

@@ -248,7 +248,7 @@ static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){
 static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){
  if( nSize>pCsr->nStat ){
    struct Fts3auxColstats *aNew;
-    aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, 
+    aNew = (struct Fts3auxColstats *)sqlite3_realloc64(pCsr->aStat, 
        sizeof(struct Fts3auxColstats) * nSize
    );
    if( aNew==0 ) return SQLITE_NOMEM;
@@ -416,15 +416,15 @@ static int fts3auxFilterMethod(
    assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) );
    if( zStr ){
      pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr);
-      pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]);
      if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM;
+      pCsr->filter.nTerm = (int)strlen(pCsr->filter.zTerm);
    }
  }

  if( iLe>=0 ){
    pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe]));
-    pCsr->nStop = sqlite3_value_bytes(apVal[iLe]);
    if( pCsr->zStop==0 ) return SQLITE_NOMEM;
+    pCsr->nStop = (int)strlen(pCsr->zStop);
  }
  
  if( iLangid>=0 ){
@@ -539,7 +539,7 @@ int sqlite3Fts3InitAux(sqlite3 *db){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
  };
  int rc;                         /* Return code */

@@ -122,8 +122,8 @@ static int fts3isspace(char c){
 ** zero the memory before returning a pointer to it. If unsuccessful, 
 ** return NULL.
 */
-static void *fts3MallocZero(int nByte){
-  void *pRet = sqlite3_malloc(nByte);
+static void *fts3MallocZero(sqlite3_int64 nByte){
+  void *pRet = sqlite3_malloc64(nByte);
  if( pRet ) memset(pRet, 0, nByte);
  return pRet;
 }
@@ -198,7 +198,7 @@ static int getNextToken(
  if( rc==SQLITE_OK ){
    const char *zToken;
    int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
-    int nByte;                               /* total space to allocate */
+    sqlite3_int64 nByte;                    /* total space to allocate */

    rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
    if( rc==SQLITE_OK ){
@@ -252,8 +252,8 @@ static int getNextToken(
 ** Enlarge a memory allocation.  If an out-of-memory allocation occurs,
 ** then free the old allocation.
 */
-static void *fts3ReallocOrFree(void *pOrig, int nNew){
-  void *pRet = sqlite3_realloc(pOrig, nNew);
+static void *fts3ReallocOrFree(void *pOrig, sqlite3_int64 nNew){
+  void *pRet = sqlite3_realloc64(pOrig, nNew);
  if( !pRet ){
    sqlite3_free(pOrig);
  }
@@ -497,7 +497,6 @@ static int getNextNode(
      int nConsumed = 0;
      pParse->nNest++;
      rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed);
-      if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; }
      *pnConsumed = (int)(zInput - z) + 1 + nConsumed;
      return rc;
    }else if( *zInput==')' ){
@@ -796,7 +795,7 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){
  if( rc==SQLITE_OK ){
    if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
      Fts3Expr **apLeaf;
-      apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
+      apLeaf = (Fts3Expr **)sqlite3_malloc64(sizeof(Fts3Expr *) * nMaxDepth);
      if( 0==apLeaf ){
        rc = SQLITE_NOMEM;
      }else{
@@ -1108,34 +1107,6 @@ void sqlite3Fts3ExprFree(Fts3Expr *pDel){

 #include <stdio.h>

-/*
-** Function to query the hash-table of tokenizers (see README.tokenizers).
-*/
-static int queryTestTokenizer(
-  sqlite3 *db, 
-  const char *zName,  
-  const sqlite3_tokenizer_module **pp
-){
-  int rc;
-  sqlite3_stmt *pStmt;
-  const char zSql[] = "SELECT fts3_tokenizer(?)";
-
-  *pp = 0;
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  if( rc!=SQLITE_OK ){
-    return rc;
-  }
-
-  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
-  if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
-      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
-    }
-  }
-
-  return sqlite3_finalize(pStmt);
-}
-
 /*
 ** Return a pointer to a buffer containing a text representation of the
 ** expression passed as the first argument. The buffer is obtained from
@@ -1203,12 +1174,12 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){
 **
 **   SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
 */
-static void fts3ExprTest(
+static void fts3ExprTestCommon(
+  int bRebalance,
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
 ){
-  sqlite3_tokenizer_module const *pModule = 0;
  sqlite3_tokenizer *pTokenizer = 0;
  int rc;
  char **azCol = 0;
@@ -1218,7 +1189,9 @@ static void fts3ExprTest(
  int ii;
  Fts3Expr *pExpr;
  char *zBuf = 0;
-  sqlite3 *db = sqlite3_context_db_handle(context);
+  Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context);
+  const char *zTokenizer = 0;
+  char *zErr = 0;

  if( argc<3 ){
    sqlite3_result_error(context, 
@@ -1227,28 +1200,22 @@ static void fts3ExprTest(
    return;
  }

-  rc = queryTestTokenizer(db,
-                          (const char *)sqlite3_value_text(argv[0]), &pModule);
-  if( rc==SQLITE_NOMEM ){
-    sqlite3_result_error_nomem(context);
-    goto exprtest_out;
-  }else if( !pModule ){
-    sqlite3_result_error(context, "No such tokenizer module", -1);
-    goto exprtest_out;
+  zTokenizer = (const char*)sqlite3_value_text(argv[0]);
+  rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr);
+  if( rc!=SQLITE_OK ){
+    if( rc==SQLITE_NOMEM ){
+      sqlite3_result_error_nomem(context);
+    }else{
+      sqlite3_result_error(context, zErr, -1);
+    }
+    sqlite3_free(zErr);
+    return;
  }

-  rc = pModule->xCreate(0, 0, &pTokenizer);
-  assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
-  if( rc==SQLITE_NOMEM ){
-    sqlite3_result_error_nomem(context);
-    goto exprtest_out;
-  }
-  pTokenizer->pModule = pModule;
-
  zExpr = (const char *)sqlite3_value_text(argv[1]);
  nExpr = sqlite3_value_bytes(argv[1]);
  nCol = argc-2;
-  azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
+  azCol = (char **)sqlite3_malloc64(nCol*sizeof(char *));
  if( !azCol ){
    sqlite3_result_error_nomem(context);
    goto exprtest_out;
@@ -1257,7 +1224,7 @@ static void fts3ExprTest(
    azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
  }

-  if( sqlite3_user_data(context) ){
+  if( bRebalance ){
    char *zDummy = 0;
    rc = sqlite3Fts3ExprParse(
        pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy
@@ -1283,23 +1250,38 @@ static void fts3ExprTest(
  sqlite3Fts3ExprFree(pExpr);

 exprtest_out:
-  if( pModule && pTokenizer ){
-    rc = pModule->xDestroy(pTokenizer);
+  if( pTokenizer ){
+    rc = pTokenizer->pModule->xDestroy(pTokenizer);
  }
  sqlite3_free(azCol);
 }

+static void fts3ExprTest(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts3ExprTestCommon(0, context, argc, argv);
+}
+static void fts3ExprTestRebalance(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts3ExprTestCommon(1, context, argc, argv);
+}
+
 /*
 ** Register the query expression parser test function fts3_exprtest() 
 ** with database connection db. 
 */
-int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){
  int rc = sqlite3_create_function(
-      db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
+      db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0
  );
  if( rc==SQLITE_OK ){
    rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", 
-        -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
+        -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0
    );
  }
  return rc;
@@ -35,8 +35,8 @@
 /*
 ** Malloc and Free functions
 */
-static void *fts3HashMalloc(int n){
-  void *p = sqlite3_malloc(n);
+static void *fts3HashMalloc(sqlite3_int64 n){
+  void *p = sqlite3_malloc64(n);
  if( p ){
    memset(p, 0, n);
  }
@@ -60,7 +60,7 @@ static int icuCreate(
  if( argc>0 ){
    n = strlen(argv[0])+1;
  }
-  p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
+  p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n);
  if( !p ){
    return SQLITE_NOMEM;
  }
@@ -117,7 +117,7 @@ static int icuOpen(
    nInput = strlen(zInput);
  }
  nChar = nInput+1;
-  pCsr = (IcuCursor *)sqlite3_malloc(
+  pCsr = (IcuCursor *)sqlite3_malloc64(
      sizeof(IcuCursor) +                /* IcuCursor */
      ((nChar+3)&~3) * sizeof(UChar) +   /* IcuCursor.aChar[] */
      (nChar+1) * sizeof(int)            /* IcuCursor.aOffset[] */
@@ -128,17 +128,19 @@ struct StrBuffer {
 /*
 ** Allocate a two-slot MatchinfoBuffer object.
 */
-static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){
+static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){
  MatchinfoBuffer *pRet;
-  int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer);
-  int nStr = (int)strlen(zMatchinfo);
+  sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1)
+                           + sizeof(MatchinfoBuffer);
+  sqlite3_int64 nStr = strlen(zMatchinfo);

-  pRet = sqlite3_malloc(nByte + nStr+1);
+  pRet = sqlite3_malloc64(nByte + nStr+1);
  if( pRet ){
    memset(pRet, 0, nByte);
    pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet;
-    pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1);
-    pRet->nElem = nElem;
+    pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0]
+                                      + sizeof(u32)*((int)nElem+1);
+    pRet->nElem = (int)nElem;
    pRet->zMatchinfo = ((char*)pRet) + nByte;
    memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1);
    pRet->aRef[0] = 1;
@@ -178,7 +180,7 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){
    aOut = &p->aMatchinfo[p->nElem+2];
    xRet = fts3MIBufferFree;
  }else{
-    aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32));
+    aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32));
    if( aOut ){
      xRet = sqlite3_free;
      if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32));
@@ -429,11 +431,12 @@ static void fts3SnippetDetails(
      char *pCsr = pPhrase->pTail;
      int iCsr = pPhrase->iTail;

-      while( iCsr<(iStart+pIter->nSnippet) ){
+      while( iCsr<(iStart+pIter->nSnippet) && iCsr>=iStart ){
        int j;
-        u64 mPhrase = (u64)1 << i;
+        u64 mPhrase = (u64)1 << (i%64);
        u64 mPos = (u64)1 << (iCsr - iStart);
-        assert( iCsr>=iStart );
+        assert( iCsr>=iStart && (iCsr - iStart)<=64 );
+        assert( i>=0 );
        if( (mCover|mCovered)&mPhrase ){
          iScore++;
        }else{
@@ -475,11 +478,14 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
    int iFirst = 0;
    pPhrase->pList = pCsr;
    fts3GetDeltaPosition(&pCsr, &iFirst);
-    assert( iFirst>=0 );
-    pPhrase->pHead = pCsr;
-    pPhrase->pTail = pCsr;
-    pPhrase->iHead = iFirst;
-    pPhrase->iTail = iFirst;
+    if( iFirst<0 ){
+      rc = FTS_CORRUPT_VTAB;
+    }else{
+      pPhrase->pHead = pCsr;
+      pPhrase->pTail = pCsr;
+      pPhrase->iHead = iFirst;
+      pPhrase->iTail = iFirst;
+    }
  }else{
    assert( rc!=SQLITE_OK || (
       pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 
@@ -516,7 +522,7 @@ static int fts3BestSnippet(
  int rc;                         /* Return Code */
  int nList;                      /* Number of phrases in expression */
  SnippetIter sIter;              /* Iterates through snippet candidates */
-  int nByte;                      /* Number of bytes of space to allocate */
+  sqlite3_int64 nByte;            /* Number of bytes of space to allocate */
  int iBestScore = -1;            /* Best snippet score found so far */
  int i;                          /* Loop counter */

@@ -534,7 +540,7 @@ static int fts3BestSnippet(
  ** the required space using malloc().
  */
  nByte = sizeof(SnippetPhrase) * nList;
-  sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte);
+  sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc64(nByte);
  if( !sIter.aPhrase ){
    return SQLITE_NOMEM;
  }
@@ -554,7 +560,7 @@ static int fts3BestSnippet(
    /* Set the *pmSeen output variable. */
    for(i=0; i<nList; i++){
      if( sIter.aPhrase[i].pHead ){
-        *pmSeen |= (u64)1 << i;
+        *pmSeen |= (u64)1 << (i%64);
      }
    }

@@ -604,8 +610,8 @@ static int fts3StringAppend(
  ** appended data.
  */
  if( pStr->n+nAppend+1>=pStr->nAlloc ){
-    int nAlloc = pStr->nAlloc+nAppend+100;
-    char *zNew = sqlite3_realloc(pStr->z, nAlloc);
+    sqlite3_int64 nAlloc = pStr->nAlloc+(sqlite3_int64)nAppend+100;
+    char *zNew = sqlite3_realloc64(pStr->z, nAlloc);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
@@ -660,6 +666,7 @@ static int fts3SnippetShift(

    for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++);
    for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++);
+    assert( (nSnippet-1-nRight)<=63 && (nSnippet-1-nRight)>=0 );
    nDesired = (nLeft-nRight)/2;

    /* Ideally, the start of the snippet should be pushed forward in the
@@ -852,7 +859,7 @@ static int fts3ColumnlistCount(char **ppCollist){
 /*
 ** This function gathers 'y' or 'b' data for a single phrase.
 */
-static void fts3ExprLHits(
+static int fts3ExprLHits(
  Fts3Expr *pExpr,                /* Phrase expression node */
  MatchInfo *p                    /* Matchinfo context */
 ){
@@ -882,25 +889,29 @@ static void fts3ExprLHits(
    if( *pIter!=0x01 ) break;
    pIter++;
    pIter += fts3GetVarint32(pIter, &iCol);
+    if( iCol>=p->nCol ) return FTS_CORRUPT_VTAB;
  }
+  return SQLITE_OK;
 }

 /*
 ** Gather the results for matchinfo directives 'y' and 'b'.
 */
-static void fts3ExprLHitGather(
+static int fts3ExprLHitGather(
  Fts3Expr *pExpr,
  MatchInfo *p
 ){
+  int rc = SQLITE_OK;
  assert( (pExpr->pLeft==0)==(pExpr->pRight==0) );
  if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
    if( pExpr->pLeft ){
-      fts3ExprLHitGather(pExpr->pLeft, p);
-      fts3ExprLHitGather(pExpr->pRight, p);
+      rc = fts3ExprLHitGather(pExpr->pLeft, p);
+      if( rc==SQLITE_OK ) rc = fts3ExprLHitGather(pExpr->pRight, p);
    }else{
-      fts3ExprLHits(pExpr, p);
+      rc = fts3ExprLHits(pExpr, p);
    }
  }
+  return rc;
 }

 /*
@@ -990,8 +1001,8 @@ static int fts3MatchinfoCheck(
  return SQLITE_ERROR;
 }

-static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
-  int nVal;                       /* Number of integers output by cArg */
+static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
+  size_t nVal;                      /* Number of integers output by cArg */

  switch( cArg ){
    case FTS3_MATCHINFO_NDOC:
@@ -1027,11 +1038,15 @@ static int fts3MatchinfoSelectDoctotal(
  Fts3Table *pTab,
  sqlite3_stmt **ppStmt,
  sqlite3_int64 *pnDoc,
-  const char **paLen
+  const char **paLen,
+  const char **ppEnd
 ){
  sqlite3_stmt *pStmt;
  const char *a;
+  const char *pEnd;
  sqlite3_int64 nDoc;
+  int n;
+

  if( !*ppStmt ){
    int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
@@ -1040,12 +1055,20 @@ static int fts3MatchinfoSelectDoctotal(
  pStmt = *ppStmt;
  assert( sqlite3_data_count(pStmt)==1 );

+  n = sqlite3_column_bytes(pStmt, 0);
  a = sqlite3_column_blob(pStmt, 0);
-  a += sqlite3Fts3GetVarint(a, &nDoc);
-  if( nDoc==0 ) return FTS_CORRUPT_VTAB;
-  *pnDoc = (u32)nDoc;
+  if( a==0 ){
+    return FTS_CORRUPT_VTAB;
+  }
+  pEnd = a + n;
+  a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc);
+  if( nDoc<=0 || a>pEnd ){
+    return FTS_CORRUPT_VTAB;
+  }
+  *pnDoc = nDoc;

  if( paLen ) *paLen = a;
+  if( ppEnd ) *ppEnd = pEnd;
  return SQLITE_OK;
 }

@@ -1117,11 +1140,12 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
  int i;
  int iCol;
  int nToken = 0;
+  int rc = SQLITE_OK;

  /* Allocate and populate the array of LcsIterator objects. The array
  ** contains one element for each matchable phrase in the query.
  **/
-  aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
+  aIter = sqlite3_malloc64(sizeof(LcsIterator) * pCsr->nPhrase);
  if( !aIter ) return SQLITE_NOMEM;
  memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
  (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
@@ -1137,13 +1161,16 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
    int nLive = 0;                /* Number of iterators in aIter not at EOF */

    for(i=0; i<pInfo->nPhrase; i++){
-      int rc;
      LcsIterator *pIt = &aIter[i];
      rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead);
-      if( rc!=SQLITE_OK ) return rc;
+      if( rc!=SQLITE_OK ) goto matchinfo_lcs_out;
      if( pIt->pRead ){
        pIt->iPos = pIt->iPosOffset;
-        fts3LcsIteratorAdvance(&aIter[i]);
+        fts3LcsIteratorAdvance(pIt);
+        if( pIt->pRead==0 ){
+          rc = FTS_CORRUPT_VTAB;
+          goto matchinfo_lcs_out;
+        }
        nLive++;
      }
    }
@@ -1175,8 +1202,9 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
    pInfo->aMatchinfo[iCol] = nLcs;
  }

+ matchinfo_lcs_out:
  sqlite3_free(aIter);
-  return SQLITE_OK;
+  return rc;
 }

 /*
@@ -1221,7 +1249,7 @@ static int fts3MatchinfoValues(
      case FTS3_MATCHINFO_NDOC:
        if( bGlobal ){
          sqlite3_int64 nDoc = 0;
-          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0);
+          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0, 0);
          pInfo->aMatchinfo[0] = (u32)nDoc;
        }
        break;
@@ -1230,14 +1258,19 @@ static int fts3MatchinfoValues(
        if( bGlobal ){
          sqlite3_int64 nDoc;     /* Number of rows in table */
          const char *a;          /* Aggregate column length array */
+          const char *pEnd;       /* First byte past end of length array */

-          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a);
+          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a, &pEnd);
          if( rc==SQLITE_OK ){
            int iCol;
            for(iCol=0; iCol<pInfo->nCol; iCol++){
              u32 iVal;
              sqlite3_int64 nToken;
              a += sqlite3Fts3GetVarint(a, &nToken);
+              if( a>pEnd ){
+                rc = SQLITE_CORRUPT_VTAB;
+                break;
+              }
              iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc);
              pInfo->aMatchinfo[iCol] = iVal;
            }
@@ -1251,9 +1284,14 @@ static int fts3MatchinfoValues(
        if( rc==SQLITE_OK ){
          int iCol;
          const char *a = sqlite3_column_blob(pSelectDocsize, 0);
+          const char *pEnd = a + sqlite3_column_bytes(pSelectDocsize, 0);
          for(iCol=0; iCol<pInfo->nCol; iCol++){
            sqlite3_int64 nToken;
-            a += sqlite3Fts3GetVarint(a, &nToken);
+            a += sqlite3Fts3GetVarintBounded(a, pEnd, &nToken);
+            if( a>pEnd ){
+              rc = SQLITE_CORRUPT_VTAB;
+              break;
+            }
            pInfo->aMatchinfo[iCol] = (u32)nToken;
          }
        }
@@ -1270,9 +1308,9 @@ static int fts3MatchinfoValues(

      case FTS3_MATCHINFO_LHITS_BM:
      case FTS3_MATCHINFO_LHITS: {
-        int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
+        size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
        memset(pInfo->aMatchinfo, 0, nZero);
-        fts3ExprLHitGather(pCsr->pExpr, pInfo);
+        rc = fts3ExprLHitGather(pCsr->pExpr, pInfo);
        break;
      }

@@ -1284,7 +1322,7 @@ static int fts3MatchinfoValues(
        if( rc!=SQLITE_OK ) break;
        if( bGlobal ){
          if( pCsr->pDeferred ){
-            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
+            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0);
            if( rc!=SQLITE_OK ) break;
          }
          rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
@@ -1339,7 +1377,7 @@ static void fts3GetMatchinfo(
  ** initialize those elements that are constant for every row.
  */
  if( pCsr->pMIBuffer==0 ){
-    int nMatchinfo = 0;           /* Number of u32 elements in match-info */
+    size_t nMatchinfo = 0;        /* Number of u32 elements in match-info */
    int i;                        /* Used to iterate through zArg */

    /* Determine the number of phrases in the query */
@@ -1424,6 +1462,10 @@ void sqlite3Fts3Snippet(
    return;
  }

+  /* Limit the snippet length to 64 tokens. */
+  if( nToken<-64 ) nToken = -64;
+  if( nToken>+64 ) nToken = +64;
+
  for(nSnippet=1; 1; nSnippet++){

    int iSnip;                    /* Loop counter 0..nSnippet-1 */
@@ -1525,7 +1567,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
  nTerm = pExpr->pPhrase->nToken;
  if( pList ){
    fts3GetDeltaPosition(&pList, &iPos);
-    assert( iPos>=0 );
+    assert_fts3_nc( iPos>=0 );
  }

  for(iTerm=0; iTerm<nTerm; iTerm++){
@@ -1566,7 +1608,7 @@ void sqlite3Fts3Offsets(
  if( rc!=SQLITE_OK ) goto offsets_out;

  /* Allocate the array of TermOffset iterators. */
-  sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken);
+  sCtx.aTerm = (TermOffset *)sqlite3_malloc64(sizeof(TermOffset)*nToken);
  if( 0==sCtx.aTerm ){
    rc = SQLITE_NOMEM;
    goto offsets_out;
@@ -1635,7 +1677,7 @@ void sqlite3Fts3Offsets(
        /* All offsets for this column have been gathered. */
        rc = SQLITE_DONE;
      }else{
-        assert( iCurrent<=iMinPos );
+        assert_fts3_nc( iCurrent<=iMinPos );
        if( 0==(0xFE&*pTerm->pList) ){
          pTerm->pList = 0;
        }else{
@@ -68,9 +68,9 @@ static int fts3termConnectMethod(
  char const *zFts3;              /* Name of fts3 table */
  int nDb;                        /* Result of strlen(zDb) */
  int nFts3;                      /* Result of strlen(zFts3) */
-  int nByte;                      /* Bytes of space to allocate here */
+  sqlite3_int64 nByte;            /* Bytes of space to allocate here */
  int rc;                         /* value returned by declare_vtab() */
-  Fts3termTable *p;                /* Virtual table object to return */
+  Fts3termTable *p;               /* Virtual table object to return */
  int iIndex = 0;

  UNUSED_PARAMETER(pCtx);
@@ -96,9 +96,9 @@ static int fts3termConnectMethod(
  if( rc!=SQLITE_OK ) return rc;

  nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
-  p = (Fts3termTable *)sqlite3_malloc(nByte);
+  p = (Fts3termTable *)sqlite3_malloc64(nByte);
  if( !p ) return SQLITE_NOMEM;
-  memset(p, 0, nByte);
+  memset(p, 0, (size_t)nByte);

  p->pFts3Tab = (Fts3Table *)&p[1];
  p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
@@ -361,7 +361,7 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
  };
  int rc;                         /* Return code */

@@ -448,14 +448,14 @@ static int testTokenizerNext(
  }else{
    /* Advance to the end of the token */
    const char *pToken = p;
-    int nToken;
+    sqlite3_int64 nToken;
    while( p<pEnd && testIsTokenChar(*p) ) p++;
-    nToken = (int)(p-pToken);
+    nToken = (sqlite3_int64)(p-pToken);

    /* Copy the token into the buffer */
    if( nToken>pCsr->nBuffer ){
      sqlite3_free(pCsr->aBuffer);
-      pCsr->aBuffer = sqlite3_malloc(nToken);
+      pCsr->aBuffer = sqlite3_malloc64(nToken);
    }
    if( pCsr->aBuffer==0 ){
      rc = SQLITE_NOMEM;
@@ -471,7 +471,7 @@ static int testTokenizerNext(
      pCsr->iInput = (int)(p - pCsr->aInput);

      *ppToken = pCsr->aBuffer;
-      *pnBytes = nToken;
+      *pnBytes = (int)nToken;
      *piStartOffset = (int)(pToken - pCsr->aInput);
      *piEndOffset = (int)(p - pCsr->aInput);
      *piPosition = pCsr->iToken;
@@ -574,6 +574,33 @@ static int SQLITE_TCLAPI fts3_test_varint_cmd(
 ** End of tokenizer code.
 **************************************************************************/ 

+/*
+**      sqlite3_fts3_may_be_corrupt BOOLEAN
+**
+** Set or clear the global "may-be-corrupt" flag. Return the old value.
+*/
+static int SQLITE_TCLAPI fts3_may_be_corrupt(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int bOld = sqlite3_fts3_may_be_corrupt;
+
+  if( objc!=2 && objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?");
+    return TCL_ERROR;
+  }
+  if( objc==2 ){
+    int bNew;
+    if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR;
+    sqlite3_fts3_may_be_corrupt = bNew;
+  }
+
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld));
+  return TCL_OK;
+}
+
 int Sqlitetestfts3_Init(Tcl_Interp *interp){
  Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
  Tcl_CreateObjCommand(interp, 
@@ -582,10 +609,12 @@ int Sqlitetestfts3_Init(Tcl_Interp *interp){
  Tcl_CreateObjCommand(
      interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0
  );
-
  Tcl_CreateObjCommand(
      interp, "fts3_test_varint", fts3_test_varint_cmd, 0, 0
  );
+  Tcl_CreateObjCommand(
+      interp, "sqlite3_fts3_may_be_corrupt", fts3_may_be_corrupt, 0, 0
+  );
  return TCL_OK;
 }
 #endif                  /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */
@@ -122,7 +122,7 @@ static int fts3tokDequoteArray(
      nByte += (int)(strlen(argv[i]) + 1);
    }

-    *pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
+    *pazDequote = azDequote = sqlite3_malloc64(sizeof(char *)*argc + nByte);
    if( azDequote==0 ){
      rc = SQLITE_NOMEM;
    }else{
@@ -346,7 +346,7 @@ static int fts3tokFilterMethod(
  if( idxNum==1 ){
    const char *zByte = (const char *)sqlite3_value_text(apVal[0]);
    int nByte = sqlite3_value_bytes(apVal[0]);
-    pCsr->zInput = sqlite3_malloc(nByte+1);
+    pCsr->zInput = sqlite3_malloc64(nByte+1);
    if( pCsr->zInput==0 ){
      rc = SQLITE_NOMEM;
    }else{
@@ -443,7 +443,7 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
  };
  int rc;                         /* Return code */

@@ -41,6 +41,17 @@ static int fts3TokenizerEnabled(sqlite3_context *context){
  return isEnabled;
 }

+/*
+** The real sqlite3_value_frombind() implementation was not added
+** until version 3.28.0 of the SQLite core.  This fake version facilitates
+** testing.
+*/
+static int bFrombindTrue = 0;
+static int sqlite3_value_frombind(sqlite3_value *NotUsed){
+  (void)NotUsed;
+  return bFrombindTrue;
+}
+
 /*
 ** Implementation of the SQL scalar function for accessing the underlying 
 ** hash table. This function may be called as follows:
@@ -79,7 +90,7 @@ static void fts3TokenizerFunc(
  nName = sqlite3_value_bytes(argv[0])+1;

  if( argc==2 ){
-    if( fts3TokenizerEnabled(context) ){
+    if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){
      void *pOld;
      int n = sqlite3_value_bytes(argv[1]);
      if( zName==0 || n!=sizeof(pPtr) ){
@@ -106,7 +117,9 @@ static void fts3TokenizerFunc(
      return;
    }
  }
-  sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+  if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){
+    sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+  }
 }

 int sqlite3Fts3IsIdChar(char c){
@@ -194,8 +207,8 @@ int sqlite3Fts3InitTokenizer(
    int iArg = 0;
    z = &z[n+1];
    while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
-      int nNew = sizeof(char *)*(iArg+1);
-      char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew);
+      sqlite3_int64 nNew = sizeof(char *)*(iArg+1);
+      char const **aNew = (const char **)sqlite3_realloc64((void *)aArg, nNew);
      if( !aNew ){
        sqlite3_free(zCopy);
        sqlite3_free((void *)aArg);
@@ -388,7 +401,9 @@ int queryTokenizer(

  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
  if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB
+     && sqlite3_column_bytes(pStmt, 0)==sizeof(*pp)
+    ){
      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
    }
  }
@@ -430,6 +445,7 @@ static void intTestFunc(
  UNUSED_PARAMETER(argv);

  /* Test the query function */
+  bFrombindTrue = 1;
  sqlite3Fts3SimpleTokenizerModule(&p1);
  rc = queryTokenizer(db, "simple", &p2);
  assert( rc==SQLITE_OK );
@@ -447,6 +463,7 @@ static void intTestFunc(
    assert( rc==SQLITE_OK );
    assert( p2==p1 );
  }
+  bFrombindTrue = 0;

  sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
 }
@@ -477,7 +494,7 @@ int sqlite3Fts3InitHashTable(
 ){
  int rc = SQLITE_OK;
  void *p = (void *)pHash;
-  const int any = SQLITE_ANY;
+  const int any = SQLITE_UTF8|SQLITE_DIRECTONLY;

 #ifdef SQLITE_TEST
  char *zTest = 0;
@@ -82,7 +82,7 @@ typedef struct unicode_cursor unicode_cursor;

 struct unicode_tokenizer {
  sqlite3_tokenizer base;
-  int bRemoveDiacritic;
+  int eRemoveDiacritic;
  int nException;
  int *aiException;
 };
@@ -155,7 +155,7 @@ static int unicodeAddExceptions(
    int *aNew;                    /* New aiException[] array */
    int nNew;                     /* Number of valid entries in array aNew[] */

-    aNew = sqlite3_realloc(p->aiException, (p->nException+nEntry)*sizeof(int));
+    aNew = sqlite3_realloc64(p->aiException,(p->nException+nEntry)*sizeof(int));
    if( aNew==0 ) return SQLITE_NOMEM;
    nNew = p->nException;

@@ -227,17 +227,20 @@ static int unicodeCreate(
  pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
  if( pNew==NULL ) return SQLITE_NOMEM;
  memset(pNew, 0, sizeof(unicode_tokenizer));
-  pNew->bRemoveDiacritic = 1;
+  pNew->eRemoveDiacritic = 1;

  for(i=0; rc==SQLITE_OK && i<nArg; i++){
    const char *z = azArg[i];
    int n = (int)strlen(z);

    if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 1;
+      pNew->eRemoveDiacritic = 1;
    }
    else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 0;
+      pNew->eRemoveDiacritic = 0;
+    }
+    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
+      pNew->eRemoveDiacritic = 2;
    }
    else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
      rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
@@ -341,7 +344,7 @@ static int unicodeNext(

    /* Grow the output buffer if required. */
    if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){
-      char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64);
+      char *zNew = sqlite3_realloc64(pCsr->zToken, pCsr->nAlloc+64);
      if( !zNew ) return SQLITE_NOMEM;
      zOut = &zNew[zOut - pCsr->zToken];
      pCsr->zToken = zNew;
@@ -350,7 +353,7 @@ static int unicodeNext(

    /* Write the folded case of the last character read to the output */
    zEnd = z;
-    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
+    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
    if( iOut ){
      WRITE_UTF8(zOut, iOut);
    }
@@ -1,5 +1,5 @@
 /*
-** 2012 May 25
+** 2012-05-25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -159,32 +159,48 @@ int sqlite3FtsUnicodeIsalnum(int c){
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int remove_diacritic(int c){
+static int remove_diacritic(int c, int bComplex){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
  };
-  char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+#define HIBIT ((unsigned char)0x80)
+  unsigned char aChar[] = {
+    '\0',      'a',       'c',       'e',       'i',       'n',       
+    'o',       'u',       'y',       'y',       'a',       'c',       
+    'd',       'e',       'e',       'g',       'h',       'i',       
+    'j',       'k',       'l',       'n',       'o',       'r',       
+    's',       't',       'u',       'u',       'w',       'y',       
+    'z',       'o',       'u',       'a',       'i',       'o',       
+    'u',       'u'|HIBIT, 'a'|HIBIT, 'g',       'k',       'o',       
+    'o'|HIBIT, 'j',       'g',       'n',       'a'|HIBIT, 'a',       
+    'e',       'i',       'o',       'r',       'u',       's',       
+    't',       'h',       'a',       'e',       'o'|HIBIT, 'o',       
+    'o'|HIBIT, 'y',       '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a',       'b',       
+    'c'|HIBIT, 'd',       'd',       'e'|HIBIT, 'e',       'e'|HIBIT, 
+    'f',       'g',       'h',       'h',       'i',       'i'|HIBIT, 
+    'k',       'l',       'l'|HIBIT, 'l',       'm',       'n',       
+    'o'|HIBIT, 'p',       'r',       'r'|HIBIT, 'r',       's',       
+    's'|HIBIT, 't',       'u',       'u'|HIBIT, 'v',       'w',       
+    'w',       'x',       'y',       'z',       'h',       't',       
+    'w',       'y',       'a',       'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 
+    'e',       'e'|HIBIT, 'e'|HIBIT, 'i',       'o',       'o'|HIBIT, 
+    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',       
  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -201,7 +217,8 @@ static int remove_diacritic(int c){
    }
  }
  assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }


@@ -214,8 +231,8 @@ int sqlite3FtsUnicodeIsdiacritic(int c){
  unsigned int mask1 = 0x000361F8;
  if( c<768 || c>817 ) return 0;
  return (c < 768+32) ?
-      (mask0 & (1 << (c-768))) :
-      (mask1 & (1 << (c-768-32)));
+      (mask0 & ((unsigned int)1 << (c-768))) :
+      (mask1 & ((unsigned int)1 << (c-768-32)));
 }


@@ -228,7 +245,7 @@ int sqlite3FtsUnicodeIsdiacritic(int c){
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
@@ -351,7 +368,9 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
      assert( ret>0 );
    }

-    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = remove_diacritic(ret, eRemoveDiacritic==2);
+    }
  }
  
  else if( c>=66560 && c<66600 ){
@@ -23,7 +23,7 @@
 #include <string.h>
 #include <assert.h>
 #include <stdlib.h>
-
+#include <stdio.h>

 #define FTS_MAX_APPENDABLE_HEIGHT 16

@@ -67,7 +67,7 @@ int test_fts3_node_chunk_threshold = (4*1024)*4;
 #endif

 /*
-** The two values that may be meaningfully bound to the :1 parameter in
+** The values that may be meaningfully bound to the :1 parameter in
 ** statements SQL_REPLACE_STAT and SQL_SELECT_STAT.
 */
 #define FTS_STAT_DOCTOTAL      0
@@ -335,7 +335,7 @@ static int fts3SqlStmt(
 ** returns zero rows.  */
 /* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' "
         "  GROUP BY level HAVING cnt>=?"
-         "  ORDER BY (level %% 1024) ASC LIMIT 1",
+         "  ORDER BY (level %% 1024) ASC, 2 DESC LIMIT 1",

 /* Estimate the upper limit on the number of leaf nodes in a new segment
 ** created by merging the oldest :2 segments from absolute level :1. See 
@@ -396,10 +396,12 @@ static int fts3SqlStmt(
  
  pStmt = p->aStmt[eStmt];
  if( !pStmt ){
+    int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB;
    char *zSql;
    if( eStmt==SQL_CONTENT_INSERT ){
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
    }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
+      f &= ~SQLITE_PREPARE_NO_VTAB;
      zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
    }else{
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
@@ -407,8 +409,7 @@ static int fts3SqlStmt(
    if( !zSql ){
      rc = SQLITE_NOMEM;
    }else{
-      rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
-                              &pStmt, NULL);
+      rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL);
      sqlite3_free(zSql);
      assert( rc==SQLITE_OK || pStmt==0 );
      p->aStmt[eStmt] = pStmt;
@@ -566,7 +567,7 @@ static sqlite3_int64 getAbsoluteLevel(
  int iLevel                      /* Level of segments */
 ){
  sqlite3_int64 iBase;            /* First absolute level for iLangid/iIndex */
-  assert( iLangid>=0 );
+  assert_fts3_nc( iLangid>=0 );
  assert( p->nIndex>0 );
  assert( iIndex>=0 && iIndex<p->nIndex );

@@ -695,7 +696,7 @@ static int fts3PendingListAppend(
  assert( !p || p->iLastDocid<=iDocid );

  if( !p || p->iLastDocid!=iDocid ){
-    sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
+    u64 iDelta = (u64)iDocid - (u64)(p ? p->iLastDocid : 0);
    if( p ){
      assert( p->nData<p->nSpace );
      assert( p->aData[p->nData]==0 );
@@ -1152,7 +1153,7 @@ static int fts3AllocateSegdirIdx(
    ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
    ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
    */
-    if( iNext>=FTS3_MERGE_COUNT ){
+    if( iNext>=MergeCount(p) ){
      fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel));
      rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel);
      *piIdx = 0;
@@ -1236,6 +1237,8 @@ int sqlite3Fts3ReadBlock(
      }
      *paBlob = aByte;
    }
+  }else if( rc==SQLITE_ERROR ){
+    rc = FTS_CORRUPT_VTAB;
  }

  return rc;
@@ -1347,7 +1350,9 @@ static int fts3SegReaderNext(

    /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf 
    ** blocks have already been traversed.  */
-    assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
+#ifdef CORRUPT_DB
+    assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock || CORRUPT_DB );
+#endif
    if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
      return SQLITE_OK;
    }
@@ -1374,15 +1379,19 @@ static int fts3SegReaderNext(
  ** safe (no risk of overread) even if the node data is corrupted. */
  pNext += fts3GetVarint32(pNext, &nPrefix);
  pNext += fts3GetVarint32(pNext, &nSuffix);
-  if( nPrefix<0 || nSuffix<=0 
-   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
+  if( nSuffix<=0 
+   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
+   || nPrefix>pReader->nTerm
  ){
    return FTS_CORRUPT_VTAB;
  }

-  if( nPrefix+nSuffix>pReader->nTermAlloc ){
-    int nNew = (nPrefix+nSuffix)*2;
-    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
+  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
+  ** overflow - hence the (i64) casts.  */
+  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
+    i64 nNew = ((i64)nPrefix+nSuffix)*2;
+    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
@@ -1404,7 +1413,7 @@ static int fts3SegReaderNext(
  ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  ** of these statements is untrue, then the data structure is corrupt.
  */
-  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
+  if( pReader->nDoclist > pReader->nNode-(pReader->aDoclist-pReader->aNode)
   || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  ){
    return FTS_CORRUPT_VTAB;
@@ -1522,18 +1531,18 @@ static int fts3SegReaderNextDocid(
    }else{
      rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
      if( rc==SQLITE_OK ){
-        sqlite3_int64 iDelta;
-        pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
+        u64 iDelta;
+        pReader->pOffsetList = p + sqlite3Fts3GetVarintU(p, &iDelta);
        if( pTab->bDescIdx ){
-          pReader->iDocid -= iDelta;
+          pReader->iDocid = (i64)((u64)pReader->iDocid - iDelta);
        }else{
-          pReader->iDocid += iDelta;
+          pReader->iDocid = (i64)((u64)pReader->iDocid + iDelta);
        }
      }
    }
  }

-  return SQLITE_OK;
+  return rc;
 }


@@ -1604,8 +1613,13 @@ int sqlite3Fts3SegReaderNew(
  Fts3SegReader *pReader;         /* Newly allocated SegReader object */
  int nExtra = 0;                 /* Bytes to allocate segment root node */

-  assert( iStartLeaf<=iEndLeaf );
+  assert( zRoot!=0 || nRoot==0 );
+#ifdef CORRUPT_DB
+  assert( zRoot!=0 || CORRUPT_DB );
+#endif
+
  if( iStartLeaf==0 ){
+    if( iEndLeaf!=0 ) return FTS_CORRUPT_VTAB;
    nExtra = nRoot + FTS3_NODE_PADDING;
  }

@@ -1625,7 +1639,7 @@ int sqlite3Fts3SegReaderNew(
    pReader->aNode = (char *)&pReader[1];
    pReader->rootOnly = 1;
    pReader->nNode = nRoot;
-    memcpy(pReader->aNode, zRoot, nRoot);
+    if( nRoot ) memcpy(pReader->aNode, zRoot, nRoot);
    memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING);
  }else{
    pReader->iCurrentBlock = iStartLeaf-1;
@@ -1740,8 +1754,9 @@ int sqlite3Fts3SegReaderPending(
  }

  if( nElem>0 ){
-    int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *);
-    pReader = (Fts3SegReader *)sqlite3_malloc(nByte);
+    sqlite3_int64 nByte;
+    nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *);
+    pReader = (Fts3SegReader *)sqlite3_malloc64(nByte);
    if( !pReader ){
      rc = SQLITE_NOMEM;
    }else{
@@ -1908,6 +1923,7 @@ static int fts3WriteSegment(
    sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
    sqlite3_step(pStmt);
    rc = sqlite3_reset(pStmt);
+    sqlite3_bind_null(pStmt, 2);
  }
  return rc;
 }
@@ -1964,6 +1980,7 @@ static int fts3WriteSegdir(
    sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
    sqlite3_step(pStmt);
    rc = sqlite3_reset(pStmt);
+    sqlite3_bind_null(pStmt, 6);
  }
  return rc;
 }
@@ -2015,6 +2032,11 @@ static int fts3NodeAddTerm(
    nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm);
    nSuffix = nTerm-nPrefix;

+    /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of 
+    ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when
+    ** compared with BINARY collation. This indicates corruption.  */
+    if( nSuffix<=0 ) return FTS_CORRUPT_VTAB;
+
    nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix;
    if( nReq<=p->nNodeSize || !pTree->zTerm ){

@@ -2243,6 +2265,11 @@ static int fts3SegWriterAdd(
  nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm);
  nSuffix = nTerm-nPrefix;

+  /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of 
+  ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when
+  ** compared with BINARY collation. This indicates corruption.  */
+  if( nSuffix<=0 ) return FTS_CORRUPT_VTAB;
+
  /* Figure out how many bytes are required by this new entry */
  nReq = sqlite3Fts3VarintLen(nPrefix) +    /* varint containing prefix size */
    sqlite3Fts3VarintLen(nSuffix) +         /* varint containing suffix size */
@@ -2254,6 +2281,7 @@ static int fts3SegWriterAdd(
    int rc;

    /* The current leaf node is full. Write it out to the database. */
+    if( pWriter->iFree==LARGEST_INT64 ) return FTS_CORRUPT_VTAB;
    rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
    if( rc!=SQLITE_OK ) return rc;
    p->nLeafAdd++;
@@ -2303,9 +2331,11 @@ static int fts3SegWriterAdd(
  /* Append the prefix-compressed term and doclist to the buffer. */
  nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix);
  nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix);
+  assert( nSuffix>0 );
  memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix);
  nData += nSuffix;
  nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist);
+  assert( nDoclist>0 );
  memcpy(&pWriter->aData[nData], aDoclist, nDoclist);
  pWriter->nData = nData + nDoclist;

@@ -2325,6 +2355,7 @@ static int fts3SegWriterAdd(
      pWriter->zTerm = zNew;
    }
    assert( pWriter->zTerm==pWriter->zMalloc );
+    assert( nTerm>0 );
    memcpy(pWriter->zTerm, zTerm, nTerm);
  }else{
    pWriter->zTerm = (char *)zTerm;
@@ -2599,14 +2630,14 @@ static void fts3ColumnFilter(

    nList -= (int)(p - pList);
    pList = p;
-    if( nList==0 ){
+    if( nList<=0 ){
      break;
    }
    p = &pList[1];
    p += fts3GetVarint32(p, &iCurrent);
  }

-  if( bZero && &pList[nList]!=pEnd ){
+  if( bZero && (pEnd - &pList[nList])>0){
    memset(&pList[nList], 0, pEnd - &pList[nList]);
  }
  *ppList = pList;
@@ -2633,6 +2664,7 @@ static int fts3MsrBufferData(
    pMsr->aBuffer = pNew;
  }

+  assert( nList>0 );
  memcpy(pMsr->aBuffer, pList, nList);
  return SQLITE_OK;
 }
@@ -2946,12 +2978,12 @@ int sqlite3Fts3SegReaderStep(
          ** doclist. */
          sqlite3_int64 iDelta;
          if( p->bDescIdx && nDoclist>0 ){
-            iDelta = iPrev - iDocid;
+            if( iPrev<=iDocid ) return FTS_CORRUPT_VTAB;
+            iDelta = (i64)((u64)iPrev - (u64)iDocid);
          }else{
-            iDelta = iDocid - iPrev;
+            if( nDoclist>0 && iPrev>=iDocid ) return FTS_CORRUPT_VTAB;
+            iDelta = (i64)((u64)iDocid - (u64)iPrev);
          }
-          assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) );
-          assert( nDoclist>0 || iDelta==iDocid );

          nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
          if( nDoclist+nByte>pCsr->nBuffer ){
@@ -3216,8 +3248,10 @@ static int fts3SegmentMerge(
  if( rc!=SQLITE_OK ) goto finished;

  assert( csr.nSegment>0 );
-  assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
-  assert( iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) );
+  assert_fts3_nc( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
+  assert_fts3_nc( 
+    iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) 
+  );

  memset(&filter, 0, sizeof(Fts3SegFilter));
  filter.flags = FTS3_SEGMENT_REQUIRE_POS;
@@ -3231,7 +3265,7 @@ static int fts3SegmentMerge(
        csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
  }
  if( rc!=SQLITE_OK ) goto finished;
-  assert( pWriter || bIgnoreEmpty );
+  assert_fts3_nc( pWriter || bIgnoreEmpty );

  if( iLevel!=FTS3_SEGCURSOR_PENDING ){
    rc = fts3DeleteSegdir(
@@ -3316,14 +3350,16 @@ static void fts3DecodeIntArray(
  const char *zBuf,  /* The BLOB containing the varints */
  int nBuf           /* size of the BLOB */
 ){
-  int i, j;
-  UNUSED_PARAMETER(nBuf);
-  for(i=j=0; i<N; i++){
-    sqlite3_int64 x;
-    j += sqlite3Fts3GetVarint(&zBuf[j], &x);
-    assert(j<=nBuf);
-    a[i] = (u32)(x & 0xffffffff);
+  int i = 0;
+  if( nBuf && (zBuf[nBuf-1]&0x80)==0 ){
+    int j;
+    for(i=j=0; i<N && j<nBuf; i++){
+      sqlite3_int64 x;
+      j += sqlite3Fts3GetVarint(&zBuf[j], &x);
+      a[i] = (u32)(x & 0xffffffff);
+    }
  }
+  while( i<N ) a[i++] = 0;
 }

 /*
@@ -3342,7 +3378,7 @@ static void fts3InsertDocsize(
  int rc;                  /* Result code from subfunctions */

  if( *pRC ) return;
-  pBlob = sqlite3_malloc( 10*p->nColumn );
+  pBlob = sqlite3_malloc64( 10*(sqlite3_int64)p->nColumn );
  if( pBlob==0 ){
    *pRC = SQLITE_NOMEM;
    return;
@@ -3392,7 +3428,7 @@ static void fts3UpdateDocTotals(
  const int nStat = p->nColumn+2;

  if( *pRC ) return;
-  a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
+  a = sqlite3_malloc64( (sizeof(u32)+10)*(sqlite3_int64)nStat );
  if( a==0 ){
    *pRC = SQLITE_NOMEM;
    return;
@@ -3443,6 +3479,7 @@ static void fts3UpdateDocTotals(
  sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
  sqlite3_step(pStmt);
  *pRC = sqlite3_reset(pStmt);
+  sqlite3_bind_null(pStmt, 2);
  sqlite3_free(a);
 }

@@ -3455,7 +3492,10 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
  int rc;
  sqlite3_stmt *pAllLangid = 0;

-  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
+  rc = sqlite3Fts3PendingTermsFlush(p);
+  if( rc==SQLITE_OK ){
+    rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
+  }
  if( rc==SQLITE_OK ){
    int rc2;
    sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
@@ -3476,7 +3516,6 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
  }

  sqlite3Fts3SegmentsClose(p);
-  sqlite3Fts3PendingTermsClear(p);

  return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
 }
@@ -3512,8 +3551,8 @@ static int fts3DoRebuild(Fts3Table *p){
    }

    if( rc==SQLITE_OK ){
-      int nByte = sizeof(u32) * (p->nColumn+1)*3;
-      aSz = (u32 *)sqlite3_malloc(nByte);
+      sqlite3_int64 nByte = sizeof(u32) * ((sqlite3_int64)p->nColumn+1)*3;
+      aSz = (u32 *)sqlite3_malloc64(nByte);
      if( aSz==0 ){
        rc = SQLITE_NOMEM;
      }else{
@@ -3579,12 +3618,12 @@ static int fts3IncrmergeCsr(
 ){
  int rc;                         /* Return Code */
  sqlite3_stmt *pStmt = 0;        /* Statement used to read %_segdir entry */  
-  int nByte;                      /* Bytes allocated at pCsr->apSegment[] */
+  sqlite3_int64 nByte;            /* Bytes allocated at pCsr->apSegment[] */

  /* Allocate space for the Fts3MultiSegReader.aCsr[] array */
  memset(pCsr, 0, sizeof(*pCsr));
  nByte = sizeof(Fts3SegReader *) * nSeg;
-  pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);
+  pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc64(nByte);

  if( pCsr->apSegment==0 ){
    rc = SQLITE_NOMEM;
@@ -3727,6 +3766,9 @@ static int nodeReaderNext(NodeReader *p){
    }
    p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);

+    if( nPrefix>p->term.n || nSuffix>p->nNode-p->iOff || nSuffix==0 ){
+      return FTS_CORRUPT_VTAB;
+    }
    blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
    if( rc==SQLITE_OK ){
      memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
@@ -3734,14 +3776,16 @@ static int nodeReaderNext(NodeReader *p){
      p->iOff += nSuffix;
      if( p->iChild==0 ){
        p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
+        if( (p->nNode-p->iOff)<p->nDoclist ){
+          return FTS_CORRUPT_VTAB;
+        }
        p->aDoclist = &p->aNode[p->iOff];
        p->iOff += p->nDoclist;
      }
    }
  }

-  assert( p->iOff<=p->nNode );
-
+  assert_fts3_nc( p->iOff<=p->nNode );
  return rc;
 }

@@ -3765,14 +3809,14 @@ static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){
  p->nNode = nNode;

  /* Figure out if this is a leaf or an internal node. */
-  if( p->aNode[0] ){
+  if( aNode && aNode[0] ){
    /* An internal node. */
    p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild);
  }else{
    p->iOff = 1;
  }

-  return nodeReaderNext(p);
+  return aNode ? nodeReaderNext(p) : SQLITE_OK;
 }

 /*
@@ -3809,6 +3853,7 @@ static int fts3IncrmergePush(
    ** be added to.  */
    nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm);
    nSuffix = nTerm - nPrefix;
+    if(nSuffix<=0 ) return FTS_CORRUPT_VTAB;
    nSpace  = sqlite3Fts3VarintLen(nPrefix);
    nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;

@@ -3902,13 +3947,14 @@ static int fts3AppendToNode(
  /* Node must have already been started. There must be a doclist for a
  ** leaf node, and there must not be a doclist for an internal node.  */
  assert( pNode->n>0 );
-  assert( (pNode->a[0]=='\0')==(aDoclist!=0) );
+  assert_fts3_nc( (pNode->a[0]=='\0')==(aDoclist!=0) );

  blobGrowBuffer(pPrev, nTerm, &rc);
  if( rc!=SQLITE_OK ) return rc;

  nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm);
  nSuffix = nTerm - nPrefix;
+  if( nSuffix<=0 ) return FTS_CORRUPT_VTAB;
  memcpy(pPrev->a, zTerm, nTerm);
  pPrev->n = nTerm;

@@ -4118,7 +4164,7 @@ static int fts3TermCmp(
  int nCmp = MIN(nLhs, nRhs);
  int res;

-  res = memcmp(zLhs, zRhs, nCmp);
+  res = (nCmp ? memcmp(zLhs, zRhs, nCmp) : 0);
  if( res==0 ) res = nLhs - nRhs;

  return res;
@@ -4202,6 +4248,10 @@ static int fts3IncrmergeLoad(
      pWriter->bNoLeafData = (pWriter->nLeafData==0);
      nRoot = sqlite3_column_bytes(pSelect, 4);
      aRoot = sqlite3_column_blob(pSelect, 4);
+      if( aRoot==0 ){
+        sqlite3_reset(pSelect);
+        return nRoot ? SQLITE_NOMEM : FTS_CORRUPT_VTAB;
+      }
    }else{
      return sqlite3_reset(pSelect);
    }
@@ -4237,6 +4287,10 @@ static int fts3IncrmergeLoad(
      int i;
      int nHeight = (int)aRoot[0];
      NodeWriter *pNode;
+      if( nHeight<1 || nHeight>FTS_MAX_APPENDABLE_HEIGHT ){
+        sqlite3_reset(pSelect);
+        return FTS_CORRUPT_VTAB;
+      }

      pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT;
      pWriter->iStart = iStart;
@@ -4250,34 +4304,42 @@ static int fts3IncrmergeLoad(

      pNode = &pWriter->aNodeWriter[nHeight];
      pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight;
-      blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc);
+      blobGrowBuffer(&pNode->block, 
+          MAX(nRoot, p->nNodeSize)+FTS3_NODE_PADDING, &rc
+      );
      if( rc==SQLITE_OK ){
        memcpy(pNode->block.a, aRoot, nRoot);
        pNode->block.n = nRoot;
+        memset(&pNode->block.a[nRoot], 0, FTS3_NODE_PADDING);
      }

      for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){
        NodeReader reader;
        pNode = &pWriter->aNodeWriter[i];

-        rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n);
-        while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader);
-        blobGrowBuffer(&pNode->key, reader.term.n, &rc);
-        if( rc==SQLITE_OK ){
-          memcpy(pNode->key.a, reader.term.a, reader.term.n);
-          pNode->key.n = reader.term.n;
-          if( i>0 ){
-            char *aBlock = 0;
-            int nBlock = 0;
-            pNode = &pWriter->aNodeWriter[i-1];
-            pNode->iBlock = reader.iChild;
-            rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0);
-            blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc);
-            if( rc==SQLITE_OK ){
-              memcpy(pNode->block.a, aBlock, nBlock);
-              pNode->block.n = nBlock;
+        if( pNode->block.a){
+          rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n);
+          while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader);
+          blobGrowBuffer(&pNode->key, reader.term.n, &rc);
+          if( rc==SQLITE_OK ){
+            memcpy(pNode->key.a, reader.term.a, reader.term.n);
+            pNode->key.n = reader.term.n;
+            if( i>0 ){
+              char *aBlock = 0;
+              int nBlock = 0;
+              pNode = &pWriter->aNodeWriter[i-1];
+              pNode->iBlock = reader.iChild;
+              rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0);
+              blobGrowBuffer(&pNode->block, 
+                  MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc
+              );
+              if( rc==SQLITE_OK ){
+                memcpy(pNode->block.a, aBlock, nBlock);
+                pNode->block.n = nBlock;
+                memset(&pNode->block.a[nBlock], 0, FTS3_NODE_PADDING);
+              }
+              sqlite3_free(aBlock);
            }
-            sqlite3_free(aBlock);
          }
        }
        nodeReaderRelease(&reader);
@@ -4520,7 +4582,10 @@ static int fts3TruncateNode(
  NodeReader reader;              /* Reader object */
  Blob prev = {0, 0, 0};          /* Previous term written to new node */
  int rc = SQLITE_OK;             /* Return code */
-  int bLeaf = aNode[0]=='\0';     /* True for a leaf node */
+  int bLeaf;                       /* True for a leaf node */
+
+  if( nNode<1 ) return FTS_CORRUPT_VTAB;
+  bLeaf = aNode[0]=='\0';

  /* Allocate required output space */
  blobGrowBuffer(pNew, nNode, &rc);
@@ -4631,6 +4696,7 @@ static int fts3TruncateSegment(
      sqlite3_bind_int(pChomp, 4, iIdx);
      sqlite3_step(pChomp);
      rc = sqlite3_reset(pChomp);
+      sqlite3_bind_null(pChomp, 2);
    }
  }

@@ -4710,6 +4776,7 @@ static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){
    sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
+    sqlite3_bind_null(pReplace, 2);
  }

  return rc;
@@ -4784,13 +4851,17 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){
  const int nHint = pHint->n;
  int i;

-  i = pHint->n-2;
+  i = pHint->n-1;
+  if( (pHint->a[i] & 0x80) ) return FTS_CORRUPT_VTAB;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
+  if( i==0 ) return FTS_CORRUPT_VTAB;
+  i--;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;

  pHint->n = i;
  i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
  i += fts3GetVarint32(&pHint->a[i], pnInput);
+  assert( i<=nHint );
  if( i!=nHint ) return FTS_CORRUPT_VTAB;

  return SQLITE_OK;
@@ -4860,8 +4931,14 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){

      rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg);
      if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){
+        /* Based on the scan in the block above, it is known that there
+        ** are no levels with a relative level smaller than that of
+        ** iAbsLevel with more than nSeg segments, or if nSeg is -1, 
+        ** no levels with more than nMin segments. Use this to limit the
+        ** value of nHintSeg to avoid a large memory allocation in case the 
+        ** merge-hint is corrupt*/
        iAbsLevel = iHintAbsLevel;
-        nSeg = nHintSeg;
+        nSeg = MIN(MAX(nMin,nSeg), nHintSeg);
        bUseHint = 1;
        bDirtyHint = 1;
      }else{
@@ -4874,7 +4951,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
    /* If nSeg is less that zero, then there is no level with at least
    ** nMin segments and no hint in the %_stat table. No work to do.
    ** Exit early in this case.  */
-    if( nSeg<0 ) break;
+    if( nSeg<=0 ) break;

    /* Open a cursor to iterate through the contents of the oldest nSeg 
    ** indexes of absolute level iAbsLevel. If this cursor is opened using 
@@ -4902,8 +4979,15 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
    }
    if( SQLITE_OK==rc && pCsr->nSegment==nSeg
     && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
-     && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
    ){
+      int bEmpty = 0;
+      rc = sqlite3Fts3SegReaderStep(p, pCsr);
+      if( rc==SQLITE_OK ){
+        bEmpty = 1;
+      }else if( rc!=SQLITE_ROW ){
+        sqlite3Fts3SegReaderFinish(pCsr);
+        break;
+      }
      if( bUseHint && iIdx>0 ){
        const char *zKey = pCsr->zTerm;
        int nKey = pCsr->nTerm;
@@ -4914,11 +4998,13 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){

      if( rc==SQLITE_OK && pWriter->nLeafEst ){
        fts3LogMerge(nSeg, iAbsLevel);
-        do {
-          rc = fts3IncrmergeAppend(p, pWriter, pCsr);
-          if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
-          if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
-        }while( rc==SQLITE_ROW );
+        if( bEmpty==0 ){
+          do {
+            rc = fts3IncrmergeAppend(p, pWriter, pCsr);
+            if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
+            if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
+          }while( rc==SQLITE_ROW );
+        }

        /* Update or delete the input segments */
        if( rc==SQLITE_OK ){
@@ -4983,7 +5069,7 @@ static int fts3DoIncrmerge(
  const char *zParam              /* Nul-terminated string containing "A,B" */
 ){
  int rc;
-  int nMin = (FTS3_MERGE_COUNT / 2);
+  int nMin = (MergeCount(p) / 2);
  int nMerge = 0;
  const char *z = zParam;

@@ -5028,7 +5114,7 @@ static int fts3DoAutoincrmerge(
  int rc = SQLITE_OK;
  sqlite3_stmt *pStmt = 0;
  p->nAutoincrmerge = fts3Getint(&zParam);
-  if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){
+  if( p->nAutoincrmerge==1 || p->nAutoincrmerge>MergeCount(p) ){
    p->nAutoincrmerge = 8;
  }
  if( !p->bHasStat ){
@@ -5111,12 +5197,12 @@ static u64 fts3ChecksumIndex(

      i64 iDocid = 0;
      i64 iCol = 0;
-      i64 iPos = 0;
+      u64 iPos = 0;

      pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid);
      while( pCsr<pEnd ){
-        i64 iVal = 0;
-        pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
+        u64 iVal = 0;
+        pCsr += sqlite3Fts3GetVarintU(pCsr, &iVal);
        if( pCsr<pEnd ){
          if( iVal==0 || iVal==1 ){
            iCol = 0;
@@ -5124,8 +5210,12 @@ static u64 fts3ChecksumIndex(
            if( iVal ){
              pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
            }else{
-              pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
-              iDocid += iVal;
+              pCsr += sqlite3Fts3GetVarintU(pCsr, &iVal);
+              if( p->bDescIdx ){
+                iDocid = (i64)((u64)iDocid - iVal);
+              }else{
+                iDocid = (i64)((u64)iDocid + iVal);
+              }
            }
          }else{
            iPos += (iVal - 2);
@@ -5198,10 +5288,9 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
      for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
        if( p->abNotindexed[iCol]==0 ){
          const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1);
-          int nText = sqlite3_column_bytes(pStmt, iCol+1);
          sqlite3_tokenizer_cursor *pT = 0;

-          rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT);
+          rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, -1, &pT);
          while( rc==SQLITE_OK ){
            char const *zToken;       /* Buffer containing token */
            int nToken = 0;           /* Number of bytes in token */
@@ -5286,7 +5375,7 @@ static int fts3DoIntegrityCheck(
 ** meaningful value to insert is the text 'optimize'.
 */
 static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
-  int rc;                         /* Return Code */
+  int rc = SQLITE_ERROR;           /* Return Code */
  const char *zVal = (const char *)sqlite3_value_text(pVal);
  int nVal = sqlite3_value_bytes(pVal);

@@ -5302,21 +5391,27 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
    rc = fts3DoIncrmerge(p, &zVal[6]);
  }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
    rc = fts3DoAutoincrmerge(p, &zVal[10]);
-#ifdef SQLITE_TEST
-  }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
-    p->nNodeSize = atoi(&zVal[9]);
-    rc = SQLITE_OK;
-  }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
-    p->nMaxPendingData = atoi(&zVal[11]);
-    rc = SQLITE_OK;
-  }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){
-    p->bNoIncrDoclist = atoi(&zVal[21]);
-    rc = SQLITE_OK;
-#endif
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
  }else{
-    rc = SQLITE_ERROR;
+    int v;
+    if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
+      v = atoi(&zVal[9]);
+      if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v;
+      rc = SQLITE_OK;
+    }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
+      v = atoi(&zVal[11]);
+      if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v;
+      rc = SQLITE_OK;
+    }else if( nVal>21 && 0==sqlite3_strnicmp(zVal,"test-no-incr-doclist=",21) ){
+      p->bNoIncrDoclist = atoi(&zVal[21]);
+      rc = SQLITE_OK;
+    }else if( nVal>11 && 0==sqlite3_strnicmp(zVal,"mergecount=",11) ){
+      v = atoi(&zVal[11]);
+      if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v;
+      rc = SQLITE_OK;
+    }
+#endif
  }
-
  return rc;
 }

@@ -5524,7 +5619,6 @@ int sqlite3Fts3UpdateMethod(
 ){
  Fts3Table *p = (Fts3Table *)pVtab;
  int rc = SQLITE_OK;             /* Return Code */
-  int isRemove = 0;               /* True for an UPDATE or DELETE */
  u32 *aSzIns = 0;                /* Sizes of inserted documents */
  u32 *aSzDel = 0;                /* Sizes of deleted documents */
  int nChng = 0;                  /* Net change in number of documents */
@@ -5558,7 +5652,7 @@ int sqlite3Fts3UpdateMethod(
  }

  /* Allocate space to hold the change in document sizes */
-  aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 );
+  aSzDel = sqlite3_malloc64(sizeof(aSzDel[0])*((sqlite3_int64)p->nColumn+1)*2);
  if( aSzDel==0 ){
    rc = SQLITE_NOMEM;
    goto update_out;
@@ -5622,7 +5716,6 @@ int sqlite3Fts3UpdateMethod(
  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
    rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
-    isRemove = 1;
  }
  
  /* If this is an INSERT or UPDATE operation, insert the new record. */
@@ -5634,7 +5727,7 @@ int sqlite3Fts3UpdateMethod(
        rc = FTS_CORRUPT_VTAB;
      }
    }
-    if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
+    if( rc==SQLITE_OK ){
      rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
    }
    if( rc==SQLITE_OK ){
@@ -9,11 +9,12 @@ proc print_rd {map} {
  set nRange 1
  set iFirst  [lindex $map 0 0]
  set cPrev   [lindex $map 0 1]
+  set fPrev   [lindex $map 0 2]

  foreach m [lrange $map 1 end] {
-    foreach {i c} $m {}
+    foreach {i c f} $m {}

-    if {$cPrev == $c} {
+    if {$cPrev == $c && $fPrev==$f} {
      for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
        if {[info exists tl_lookup_table($j)]==0} break
      }
@@ -29,13 +30,16 @@ proc print_rd {map} {

    lappend lRange [list $iFirst $nRange]
    lappend aChar  $cPrev
+    lappend aFlag  $fPrev

    set iFirst $i
    set cPrev  $c
+    set fPrev  $f
    set nRange 1
  }
  lappend lRange [list $iFirst $nRange]
  lappend aChar $cPrev
+  lappend aFlag $fPrev

  puts "/*"
  puts "** If the argument is a codepoint corresponding to a lowercase letter"
@@ -45,7 +49,7 @@ proc print_rd {map} {
  puts "** E\"). The resuls of passing a codepoint that corresponds to an"
  puts "** uppercase letter are undefined."
  puts "*/"
-  puts "static int ${::remove_diacritic}(int c)\{"
+  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
  puts "  unsigned short aDia\[\] = \{"
  puts -nonewline "        0, "
  set i 1
@@ -59,14 +63,19 @@ proc print_rd {map} {
  }
  puts ""
  puts "  \};"
-  puts "  char aChar\[\] = \{"
-  puts -nonewline "    '\\0', "
+  puts "#define HIBIT ((unsigned char)0x80)"
+  puts "  unsigned char aChar\[\] = \{"
+  puts -nonewline "    '\\0',      "
  set i 1
-  foreach c $aChar {
-    set str "'$c',  "
-    if {$c == ""} { set str "'\\0', " }
+  foreach c $aChar f $aFlag {
+    if { $f } {
+      set str "'$c'|HIBIT, "
+    } else {
+      set str "'$c',       "
+    }
+    if {$c == ""} { set str "'\\0',      " }

-    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
+    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
    incr i
    puts -nonewline "$str"
  }
@@ -87,7 +96,8 @@ proc print_rd {map} {
    }
  }
  assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
  puts "\}"
 }

@@ -95,7 +105,8 @@ proc print_isdiacritic {zFunc map} {

  set lCode [list]
  foreach m $map {
-    foreach {code char} $m {}
+    foreach {code char flag} $m {}
+    if {$flag} continue
    if {$code && $char == ""} { lappend lCode $code }
  }
  set lCode [lsort -integer $lCode]
@@ -124,8 +135,8 @@ proc print_isdiacritic {zFunc map} {

  puts "  if( c<$iFirst || c>$iLast ) return 0;"
  puts "  return (c < $iFirst+32) ?"
-  puts "      (mask0 & (1 << (c-$iFirst))) :"
-  puts "      (mask1 & (1 << (c-$iFirst-32)));"
+  puts "      (mask0 & ((unsigned int)1 << (c-$iFirst))) :"
+  puts "      (mask1 & ((unsigned int)1 << (c-$iFirst-32)));"
  puts "\}"
 }

@@ -472,7 +483,7 @@ proc print_fold {zFunc} {
  puts "** The results are undefined if the value passed to this function"
  puts "** is less than zero."
  puts "*/"
-  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
+  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"

  set liOff [tl_generate_ioff_table $lRecord]
  tl_print_table_header
@@ -516,7 +527,9 @@ proc print_fold {zFunc} {
      assert( ret>0 );
    }

-    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
+    if( eRemoveDiacritic ){
+      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
+    }
  }
  }]

@@ -529,6 +542,259 @@ proc print_fold {zFunc} {
  puts "\}"
 }

+proc code {txt} {
+  set txt [string trimright $txt]
+  set txt [string trimleft $txt "\n"]
+  set n [expr {[string length $txt] - [string length [string trim $txt]]}]
+  set ret ""
+  foreach L [split $txt "\n"] {
+    append ret "[string range $L $n end]\n"
+  }
+  return [uplevel "subst -nocommands {$ret}"]
+}
+
+proc intarray {lInt} {
+  set ret ""
+  set n [llength $lInt]
+  for {set i 0} {$i < $n} {incr i 10} {
+    append ret "\n    "
+    foreach int [lrange $lInt $i [expr $i+9]] {
+      append ret [format "%-7s" "$int, "]
+    }
+  }
+  append ret "\n  "
+  set ret
+}
+
+proc categories_switch {Cvar first lSecond} {
+  upvar $Cvar C
+  set ret ""
+  append ret "case '$first':\n"
+  append ret "          switch( zCat\[1\] ){\n"
+  foreach s $lSecond {
+    append ret "            case '$s': aArray\[$C($first$s)\] = 1; break;\n"
+  }
+  append ret "            case '*': \n"
+  foreach s $lSecond {
+    append ret "              aArray\[$C($first$s)\] = 1;\n"
+  }
+  append ret "              break;\n"
+  append ret "            default: return 1;"
+  append ret "          }\n"
+  append ret "          break;\n"
+}
+
+# Argument is a list. Each element of which is itself a list of two elements:
+#
+#   * the codepoint
+#   * the category
+#
+# List elements are sorted in order of codepoint.
+#
+proc print_categories {lMap} {
+  set categories {
+    Cc Cf Cn Cs
+    Ll Lm Lo Lt Lu
+    Mc Me Mn
+    Nd Nl No
+    Pc Pd Pe Pf Pi Po Ps
+    Sc Sk Sm So
+    Zl Zp Zs
+
+    LC Co
+  }
+
+  for {set i 0} {$i < [llength $categories]} {incr i} {
+    set C([lindex $categories $i]) [expr 1+$i]
+  }
+
+  set caseC [categories_switch C C {c f n s o}]
+  set caseL [categories_switch C L {l m o t u C}]
+  set caseM [categories_switch C M {c e n}]
+  set caseN [categories_switch C N {d l o}]
+  set caseP [categories_switch C P {c d e f i o s}]
+  set caseS [categories_switch C S {c k m o}]
+  set caseZ [categories_switch C Z {l p s}]
+
+  set nCat [expr [llength [array names C]] + 1]
+  puts [code {
+    int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
+      aArray[0] = 1;
+      switch( zCat[0] ){
+        $caseC
+        $caseL
+        $caseM
+        $caseN
+        $caseP
+        $caseS
+        $caseZ
+      }
+      return 0;
+    }
+  }]
+
+  set nRepeat 0
+  set first   [lindex $lMap 0 0]
+  set class   [lindex $lMap 0 1]
+  set prev -1
+
+  set CASE(0) "Lu"
+  set CASE(1) "Ll"
+
+  foreach m $lMap {
+    foreach {codepoint cl} $m {}
+    set codepoint [expr "0x$codepoint"]
+    if {$codepoint>=(1<<20)} continue
+
+    set bNew 0
+    if {$codepoint!=($prev+1)} {
+      set bNew 1
+    } elseif {
+      $cl==$class || ($class=="LC" && $cl==$CASE([expr $nRepeat & 0x01]))
+    } {
+      incr nRepeat
+    } elseif {$class=="Lu" && $nRepeat==1 && $cl=="Ll"} {
+      set class LC
+      incr nRepeat
+    } else {
+      set bNew 1
+    }
+    if {$bNew} {
+      lappend lEntries [list $first $class $nRepeat]
+      set nRepeat 1
+      set first $codepoint
+      set class $cl
+    }
+    set prev $codepoint
+  }
+  if {$nRepeat>0} {
+    lappend lEntries [list $first $class $nRepeat]
+  }
+
+  set aBlock [list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+  set aMap [list]
+  foreach e $lEntries {
+    foreach {cp class nRepeat} $e {}
+    set block [expr ($cp>>16)]
+    if {$block>0 && [lindex $aBlock $block]==0} {
+      for {set i 1} {$i<=$block} {incr i} {
+        if {[lindex $aBlock $i]==0} {
+          lset aBlock $i [llength $aMap]
+        }
+      }
+    }
+    lappend aMap [expr {$cp & 0xFFFF}]
+    lappend aData [expr {($nRepeat << 5) + $C($class)}]
+  }
+  for {set i 1} {$i<[llength $aBlock]} {incr i} {
+    if {[lindex $aBlock $i]==0} {
+      lset aBlock $i [llength $aMap]
+    }
+  }
+
+  set aBlockArray [intarray $aBlock]
+  set aMapArray [intarray $aMap]
+  set aDataArray [intarray $aData]
+  puts [code {
+    static u16 aFts5UnicodeBlock[] = {$aBlockArray};
+    static u16 aFts5UnicodeMap[] = {$aMapArray};
+    static u16 aFts5UnicodeData[] = {$aDataArray};
+
+    int sqlite3Fts5UnicodeCategory(u32 iCode) { 
+      int iRes = -1;
+      int iHi;
+      int iLo;
+      int ret;
+      u16 iKey;
+
+      if( iCode>=(1<<20) ){
+        return 0;
+      }
+      iLo = aFts5UnicodeBlock[(iCode>>16)];
+      iHi = aFts5UnicodeBlock[1+(iCode>>16)];
+      iKey = (iCode & 0xFFFF);
+      while( iHi>iLo ){
+        int iTest = (iHi + iLo) / 2;
+        assert( iTest>=iLo && iTest<iHi );
+        if( iKey>=aFts5UnicodeMap[iTest] ){
+          iRes = iTest;
+          iLo = iTest+1;
+        }else{
+          iHi = iTest;
+        }
+      }
+
+      if( iRes<0 ) return 0;
+      if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
+      ret = aFts5UnicodeData[iRes] & 0x1F;
+      if( ret!=$C(LC) ) return ret;
+      return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? $C(Ll) : $C(Lu);
+    }
+
+    void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
+      int i = 0;
+      int iTbl = 0;
+      while( i<128 ){
+        int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
+        int n = (aFts5UnicodeData[iTbl] >> 5) + i;
+        for(; i<128 && i<n; i++){
+          aAscii[i] = (u8)bToken;
+        }
+        iTbl++;
+      }
+    }
+  }]
+}
+
+proc print_test_categories {lMap} {
+
+  set lCP [list]
+  foreach e $lMap {
+    foreach {cp cat} $e {}
+    if {[expr 0x$cp] < (1<<20)} {
+      lappend lCP "{0x$cp, \"$cat\"}, "
+    }
+  }
+
+  set aCP "\n"
+  for {set i 0} {$i < [llength $lCP]} {incr i 4} {
+    append aCP "    [join [lrange $lCP $i $i+3]]\n"
+  }
+
+
+  puts [code {
+    static int categories_test (int *piCode){
+      struct Codepoint {
+        int iCode;
+        const char *zCat;
+      } aCP[] = {$aCP};
+      int i;
+      int iCP = 0;
+
+      for(i=0; i<1000000; i++){
+        u8 aArray[40];
+        int cat = 0;
+        int c = 0;
+        memset(aArray, 0, sizeof(aArray));
+        if( aCP[iCP].iCode==i ){
+          sqlite3Fts5UnicodeCatParse(aCP[iCP].zCat, aArray);
+          iCP++;
+        }else{
+          aArray[0] = 1;
+        }
+
+        c = sqlite3Fts5UnicodeCategory((u32)i);
+        if( aArray[c]==0 ){
+          *piCode = i;
+          return 1;
+        }
+      }
+
+      return 0;
+    }
+  }]
+}
+
 proc print_fold_test {zFunc mappings} {
  global tl_lookup_table

@@ -572,7 +838,7 @@ proc print_fold_test {zFunc mappings} {
 proc print_fileheader {} {
  puts [string trim {
 /*
-** 2012 May 25
+** 2012-05-25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -605,15 +871,21 @@ proc print_test_main {} {
  puts "#include <stdio.h>"
  puts ""
  puts "int main(int argc, char **argv)\{"
-  puts "  int r1, r2;"
+  puts "  int r1, r2, r3;"
  puts "  int code;"
+  puts "  r3 = 0;"
  puts "  r1 = isalnum_test(&code);"
  puts "  if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);"
  puts "  else printf(\"isalnum(): test passed\\n\");"
  puts "  r2 = fold_test(&code);"
  puts "  if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);"
  puts "  else printf(\"fold(): test passed\\n\");"
-  puts "  return (r1 || r2);"
+  if {$::generate_fts5_code} {
+    puts "  r3 = categories_test(&code);"
+    puts "  if( r3 ) printf(\"categories(): Problem with code %d\\n\",code);"
+    puts "  else printf(\"categories(): test passed\\n\");"
+  }
+  puts "  return (r1 || r2 || r3);"
  puts "\}"
 }

@@ -651,10 +923,18 @@ for {set i 0} {$i < [llength $argv]-2} {incr i} {

 print_fileheader

+if {$::generate_test_code} {
+  puts "typedef unsigned short int u16;"
+  puts "typedef unsigned char u8;"
+  puts "#include <string.h>"
+}
+
 # Print the isalnum() function to stdout.
 #
 set lRange [an_load_separator_ranges]
-print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+if {$generate_fts5_code==0} {
+  print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+}

 # Leave a gap between the two generated C functions.
 #
@@ -677,12 +957,21 @@ puts ""
 #
 print_fold ${function_prefix}UnicodeFold

+if {$generate_fts5_code} {
+  puts ""
+  puts ""
+  print_categories [cc_load_unicodedata_text ${unicodedata.txt}]
+}
+
 # Print the test routines and main() function to stdout, if -test 
 # was specified.
 #
 if {$::generate_test_code} {
-  print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+  if {$generate_fts5_code==0} {
+    print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+  }
  print_fold_test ${function_prefix}UnicodeFold $mappings
+  print_test_categories [cc_load_unicodedata_text ${unicodedata.txt}]
  print_test_main 
 }

@@ -7,12 +7,24 @@
 # character that it should be replaced with, or an empty string if the 
 # codepoint should simply be removed from the input. Examples:
 #
-#   { 224 a  }     (replace codepoint 224 to "a")
-#   { 769 "" }     (remove codepoint 769 from input)
+#   { 224 a  0 }     (replace codepoint 224 to "a")
+#   { 769 "" 0 }     (remove codepoint 769 from input)
 #
 # Mappings are only returned for non-upper case codepoints. It is assumed
 # that the input has already been folded to lower case.
 #
+# The third value in the list is always either 0 or 1. 0 if the 
+# UnicodeData.txt file maps the codepoint to a single ASCII character and
+# a diacritic, or 1 if the mapping is indirect. For example, consider the 
+# two entries:
+#
+# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
+# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
+#
+# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
+# diacritic). The second is an indirect mapping, as it maps to the
+# first codepoint plus 0302 (a diacritic).
+#
 proc rd_load_unicodedata_text {zName} {
  global tl_lookup_table

@@ -53,18 +65,29 @@ proc rd_load_unicodedata_text {zName} {
    set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
    set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]

+    # Filter out upper-case characters, as they will be mapped to their
+    # lower-case equivalents before this data is used.
    if {[info exists tl_lookup_table($iCode)]} continue

+    # Check if this is an indirect mapping. If so, set bIndirect to true
+    # and change $iAscii to the indirectly mappped ASCII character.
+    set bIndirect 0
+    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
+      set iAscii $mapping($iAscii)
+      set bIndirect 1
+    }
+
    if { ($iAscii >= 97 && $iAscii <= 122)
      || ($iAscii >= 65 && $iAscii <= 90)
    } {
-      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
+      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
+      set mapping($iCode) $iAscii
      set dia($iDia) 1
    }
  }

  foreach d [array names dia] {
-    lappend lRet [list $d ""]
+    lappend lRet [list $d "" 0]
  }
  set lRet [lsort -integer -index 0 $lRet]

@@ -143,4 +166,40 @@ proc tl_load_casefolding_txt {zName} {
  }
 }

+proc cc_load_unicodedata_text {zName} {
+  set fd [open $zName]
+  set lField {
+    code
+    character_name
+    general_category
+    canonical_combining_classes
+    bidirectional_category
+    character_decomposition_mapping
+    decimal_digit_value
+    digit_value
+    numeric_value
+    mirrored
+    unicode_1_name
+    iso10646_comment_field
+    uppercase_mapping
+    lowercase_mapping
+    titlecase_mapping
+  }
+  set lRet [list]
+
+  while { ![eof $fd] } {
+    set line [gets $fd]
+    if {$line == ""} continue
+
+    set fields [split $line ";"]
+    if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
+    foreach $lField $fields {}
+
+    lappend lRet [list $code $general_category]
+  }
+
+  close $fd
+  set lRet
+}
+

@@ -445,7 +445,9 @@ static int fts5HashEntrySort(
  for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
    Fts5HashEntry *pIter;
    for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
-      if( pTerm==0 || 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm) ){
+      if( pTerm==0 
+       || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
+      ){
        Fts5HashEntry *pEntry = pIter;
        pEntry->pScanNext = 0;
        for(i=0; ap[i]; i++){
@@ -591,7 +591,19 @@ do_execsql_test 22.1 {
  SELECT rowid FROM t9('a*')
 } {1}

+
+#-------------------------------------------------------------------------
+do_execsql_test 25.0 {
+  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%);
+}
+do_execsql_test 25.1 {
+  BEGIN;
+  INSERT INTO t13 VALUES('AAAA');
+SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB*');
+
+  END;
 }

+}

 finish_test
@@ -0,0 +1,63 @@
+# 2018 March 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rbucollate
+
+ifcapable !icu_collations {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+reset_db
+ 
+# Create a simple RBU database. That expects to write to a table:
+#
+#   CREATE TABLE t1(a PRIMARY KEY, b, c);
+#
+proc create_rbu1 {filename} {
+  forcedelete $filename
+  sqlite3 rbu1 $filename  
+  rbu1 eval {
+    CREATE TABLE data_t1(a, b, c, rbu_control);
+    INSERT INTO data_t1 VALUES('a', 'one', 1, 0);
+    INSERT INTO data_t1 VALUES('b', 'two', 2, 0);
+    INSERT INTO data_t1 VALUES('c', 'three', 3, 0);
+  }
+  rbu1 close
+  return $filename
+}
+
+do_execsql_test 1.0 {
+  SELECT icu_load_collation('en_US', 'my-collate');
+  CREATE TABLE t1(a COLLATE "my-collate" PRIMARY KEY, b, c);
+} {{}}
+
+do_test 1.2 {
+  create_rbu1 testrbu.db
+  sqlite3rbu rbu test.db testrbu.db
+  rbu dbMain_eval { SELECT icu_load_collation('en_US', 'my-collate') }
+  rbu dbRbu_eval { SELECT icu_load_collation('en_US', 'my-collate') }
+  while 1 {
+    set rc [rbu step]
+    if {$rc!="SQLITE_OK"} break
+  }
+  rbu close
+  db eval { SELECT * FROM t1 }
+} {a one 1 b two 2 c three 3}
+
+#forcedelete testrbu.db
+finish_test
+
@@ -399,7 +399,8 @@ struct rbu_vfs {
  sqlite3_vfs *pRealVfs;          /* Underlying VFS */
  sqlite3_mutex *mutex;           /* Mutex to protect pMain */
  sqlite3rbu *pRbu;               /* Owner RBU object */
-  rbu_file *pMain;                /* Linked list of main db files */
+  rbu_file *pMain;                /* List of main db files */
+  rbu_file *pMainRbu;             /* List of main db files with pRbu!=0 */
 };

 /*
@@ -428,6 +429,7 @@ struct rbu_file {
  const char *zWal;               /* Wal filename for this main db file */
  rbu_file *pWalFd;               /* Wal file descriptor for this main db */
  rbu_file *pMainNext;            /* Next MAIN_DB file */
+  rbu_file *pMainRbuNext;         /* Next MAIN_DB file with pRbu!=0 */
 };

 /*
@@ -1806,7 +1808,7 @@ static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){
        int iCid = sqlite3_column_int(pXInfo, 1);
        int bDesc = sqlite3_column_int(pXInfo, 3);
        const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
-        zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma, 
+        zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %Q", zCols, zComma, 
            iCid, pIter->azTblType[iCid], zCollate
        );
        zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":"");
@@ -1867,7 +1869,7 @@ static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){
        ** "PRIMARY KEY" to the imposter table column declaration. */
        zPk = "PRIMARY KEY ";
      }
-      zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s", 
+      zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %Q%s", 
          zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl,
          (pIter->abNotNull[iCol] ? " NOT NULL" : "")
      );
@@ -4016,6 +4018,69 @@ static int rbuUpdateTempSize(rbu_file *pFd, sqlite3_int64 nNew){
  return SQLITE_OK;
 }

+/*
+** Add an item to the main-db lists, if it is not already present.
+**
+** There are two main-db lists. One for all file descriptors, and one
+** for all file descriptors with rbu_file.pDb!=0. If the argument has
+** rbu_file.pDb!=0, then it is assumed to already be present on the
+** main list and is only added to the pDb!=0 list.
+*/
+static void rbuMainlistAdd(rbu_file *p){
+  rbu_vfs *pRbuVfs = p->pRbuVfs;
+  rbu_file *pIter;
+  assert( (p->openFlags & SQLITE_OPEN_MAIN_DB) );
+  sqlite3_mutex_enter(pRbuVfs->mutex);
+  if( p->pRbu==0 ){
+    for(pIter=pRbuVfs->pMain; pIter; pIter=pIter->pMainNext);
+    p->pMainNext = pRbuVfs->pMain;
+    pRbuVfs->pMain = p;
+  }else{
+    for(pIter=pRbuVfs->pMainRbu; pIter && pIter!=p; pIter=pIter->pMainRbuNext){}
+    if( pIter==0 ){
+      p->pMainRbuNext = pRbuVfs->pMainRbu;
+      pRbuVfs->pMainRbu = p;
+    }
+  }
+  sqlite3_mutex_leave(pRbuVfs->mutex);
+}
+
+/*
+** Remove an item from the main-db lists.
+*/
+static void rbuMainlistRemove(rbu_file *p){
+  rbu_file **pp;
+  sqlite3_mutex_enter(p->pRbuVfs->mutex);
+  for(pp=&p->pRbuVfs->pMain; *pp && *pp!=p; pp=&((*pp)->pMainNext)){}
+  if( *pp ) *pp = p->pMainNext;
+  p->pMainNext = 0;
+  for(pp=&p->pRbuVfs->pMainRbu; *pp && *pp!=p; pp=&((*pp)->pMainRbuNext)){}
+  if( *pp ) *pp = p->pMainRbuNext;
+  p->pMainRbuNext = 0;
+  sqlite3_mutex_leave(p->pRbuVfs->mutex);
+}
+
+/*
+** Given that zWal points to a buffer containing a wal file name passed to 
+** either the xOpen() or xAccess() VFS method, search the main-db list for
+** a file-handle opened by the same database connection on the corresponding
+** database file.
+**
+** If parameter bRbu is true, only search for file-descriptors with
+** rbu_file.pDb!=0.
+*/
+static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal, int bRbu){
+  rbu_file *pDb;
+  sqlite3_mutex_enter(pRbuVfs->mutex);
+  if( bRbu ){
+    for(pDb=pRbuVfs->pMainRbu; pDb && pDb->zWal!=zWal; pDb=pDb->pMainRbuNext){}
+  }else{
+    for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){}
+  }
+  sqlite3_mutex_leave(pRbuVfs->mutex);
+  return pDb;
+}
+
 /*
 ** Close an rbu file.
 */
@@ -4033,17 +4098,14 @@ static int rbuVfsClose(sqlite3_file *pFile){
  sqlite3_free(p->zDel);

  if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
-    rbu_file **pp;
-    sqlite3_mutex_enter(p->pRbuVfs->mutex);
-    for(pp=&p->pRbuVfs->pMain; *pp!=p; pp=&((*pp)->pMainNext));
-    *pp = p->pMainNext;
-    sqlite3_mutex_leave(p->pRbuVfs->mutex);
+    rbuMainlistRemove(p);
    rbuUnlockShm(p);
    p->pReal->pMethods->xShmUnmap(p->pReal, 0);
  }
  else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
    rbuUpdateTempSize(p, 0);
  }
+  assert( p->pMainNext==0 && p->pRbuVfs->pMain!=p );

  /* Close the underlying file handle */
  rc = p->pReal->pMethods->xClose(p->pReal);
@@ -4302,6 +4364,9 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){
      }else if( rc==SQLITE_NOTFOUND ){
        pRbu->pTargetFd = p;
        p->pRbu = pRbu;
+        if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
+          rbuMainlistAdd(p);
+        }
        if( p->pWalFd ) p->pWalFd->pRbu = pRbu;
        rc = SQLITE_OK;
      }
@@ -4463,20 +4528,6 @@ static int rbuVfsShmUnmap(sqlite3_file *pFile, int delFlag){
  return rc;
 }

-/*
-** Given that zWal points to a buffer containing a wal file name passed to 
-** either the xOpen() or xAccess() VFS method, return a pointer to the
-** file-handle opened by the same database connection on the corresponding
-** database file.
-*/
-static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){
-  rbu_file *pDb;
-  sqlite3_mutex_enter(pRbuVfs->mutex);
-  for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){}
-  sqlite3_mutex_leave(pRbuVfs->mutex);
-  return pDb;
-}
-
 /* 
 ** A main database named zName has just been opened. The following 
 ** function returns a pointer to a buffer owned by SQLite that contains
@@ -4555,7 +4606,7 @@ static int rbuVfsOpen(
      pFd->zWal = rbuMainToWal(zName, flags);
    }
    else if( flags & SQLITE_OPEN_WAL ){
-      rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName);
+      rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName, 0);
      if( pDb ){
        if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
          /* This call is to open a *-wal file. Intead, open the *-oal. This
@@ -4607,10 +4658,7 @@ static int rbuVfsOpen(
    ** mutex protected linked list of all such files.  */
    pFile->pMethods = &rbuvfs_io_methods;
    if( flags & SQLITE_OPEN_MAIN_DB ){
-      sqlite3_mutex_enter(pRbuVfs->mutex);
-      pFd->pMainNext = pRbuVfs->pMain;
-      pRbuVfs->pMain = pFd;
-      sqlite3_mutex_leave(pRbuVfs->mutex);
+      rbuMainlistAdd(pFd);
    }
  }else{
    sqlite3_free(pFd->zDel);
@@ -4658,7 +4706,7 @@ static int rbuVfsAccess(
  **      file opened instead.
  */
  if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){
-    rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath);
+    rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath, 1);
    if( pDb && pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
      if( *pResOut ){
        rc = SQLITE_CANTOPEN;
@@ -81,6 +81,7 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd(
    {"close_no_error", 2, ""},       /* 9 */
    {"temp_size_limit", 3, "LIMIT"}, /* 10 */
    {"temp_size", 2, ""},            /* 11 */
+    {"dbRbu_eval", 3, "SQL"},        /* 12 */
    {0,0,0}
  };
  int iCmd;
@@ -146,8 +147,9 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd(
      break;
    }

-    case 4: /* dbMain_eval */ {
-      sqlite3 *db = sqlite3rbu_db(pRbu, 0);
+    case 12: /* dbRbu_eval */ 
+    case 4:  /* dbMain_eval */ {
+      sqlite3 *db = sqlite3rbu_db(pRbu, (iCmd==12));
      int rc = sqlite3_exec(db, Tcl_GetString(objv[2]), 0, 0, 0);
      if( rc!=SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errmsg(db), -1));
@@ -3526,6 +3526,7 @@ static int rtreeInit(
 }


+#if defined(SQLITE_TEST)
 /*
 ** Implementation of a scalar function that decodes r-tree nodes to
 ** human readable strings. This can be used for debugging and analysis.
@@ -3587,6 +3588,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){
  
  sqlite3_result_text(ctx, zText, -1, sqlite3_free);
 }
+#endif

 /* This routine implements an SQL function that returns the "depth" parameter
 ** from the front of a blob that is an r-tree node.  For example:
@@ -4073,9 +4075,11 @@ static void rtreecheck(
 */
 int sqlite3RtreeInit(sqlite3 *db){
  const int utf8 = SQLITE_UTF8;
-  int rc;
+  int rc = SQLITE_OK;

+#if defined(SQLITE_TEST)
  rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0);
+#endif
  if( rc==SQLITE_OK ){
    rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0);
  }
@@ -1,5 +1,5 @@
-C Version\s3.22.0
-D 2018-01-22T18:45:57.681
+C Import\sall\sFTS3/4\senhancements\sand\sfixes\sthat\sexist\son\sthe\slatest\strunk\n(3.31.0-beta)\sthat\sdo\snot\srequire\sextensive\schange\sto\sthe\sSQLite\score\ninto\sthe\s3.22\sbranch.\s\sBasically,\sthe\s3.31.0\sFTS3\ssources\sare\scopied\sinto\n3.22.0,\swith\sminor\schanges\sto\swork\saround\score\senhancements\sthat\sare\snot\navailable\sin\s3.22.0.
+D 2020-01-17T14:18:48.465
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -76,35 +76,35 @@ F ext/fts2/fts2_tokenizer1.c 07e223eecb483d448313b5f1553a4f299a7fb7a1
 F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
-F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
+F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c f1c58503bc81c3dab1a70b25e146878ae40fccc716fd7c9b817995b661bc896f
+F ext/fts3/fts3.c b3192f3596592b11f964ec57a46629946d07cb762c54690487d90315384a1f4b
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a
-F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1
-F ext/fts3/fts3_expr.c dfd571a24412779ac01f25c01d888c6ef7b2d0ef
-F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60
+F ext/fts3/fts3Int.h f091030b976045e7df91af2337935952b477cdbd9f48058c44c965684484cb50
+F ext/fts3/fts3_aux.c 489e1e383868263207aef8ece4ab0a53053e4d7cf93ec48db79736140a5fb2e9
+F ext/fts3/fts3_expr.c b132af223e90e35b9f9efa9fe63d6ae737d34153a3b6066736086df8abc78a1f
+F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7
 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
-F ext/fts3/fts3_icu.c deb46f7020d87ea7a14a433fb7a7f4bef42a9652
+F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116
 F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009
-F ext/fts3/fts3_snippet.c 68ae118b0f834ea53d2b89e4087fc0f0b8c4ee4e
-F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7
-F ext/fts3/fts3_test.c 79f2a7fbb3f672fa032e5a432ca274ea3ee93c34
-F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860
-F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7
+F ext/fts3/fts3_snippet.c 052b35ad746349ffb53820379bacdb23ff3ac60d3cc13d986e56d42822ef5a9a
+F ext/fts3/fts3_term.c b529d6cceecd85a8256c4275012ac739744b38008b55f56184f35839110d0fab
+F ext/fts3/fts3_test.c 73b16e229e517c1b1f0fb8e1046182a4e5dbc8dbe6eea8a5d4353fcce7dbbf39
+F ext/fts3/fts3_tokenize_vtab.c 8ac0608b94c36980dcfaaaf63c95aba988e0e086a08b4c31563c1fb601458bd2
+F ext/fts3/fts3_tokenizer.c 3a9c6a2551b0dd7b8f70f193df4ec263b9f5541f93c4ca737358f2457817d9c1
 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
-F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d
-F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c
-F ext/fts3/fts3_write.c a3f7bf869622d1d0aa66661ba71d88e6f9646d69a2c335f40a0addf25974db47
+F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d
+F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f
+F ext/fts3/fts3_write.c 6f9dd5d774003ea81b8b32daa7d0819f9aaf01bf2b5f33498a69aab096094ed3
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
 F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
-F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c4951a0fbb60e780cb69f
-F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
+F ext/fts3/unicode/mkunicode.tcl bf7fcaa6d68e6d38223467983785d054f1cff4d9e3905dd51f6ed8801bb590d5
+F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h eda28e3a0a5d87c412e8355fe35da875b04cb389908c8eb0d867ad662adbc491
@@ -112,7 +112,7 @@ F ext/fts5/fts5_aux.c ca666a3bbe07c5a3bbe9fffaea19c935a1efaf337333e28bad7bdd1971
 F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bfd13e973ff
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 01048018d21524e2c302b063ff5c3cdcf546e03297215e577205d85b47499deb
-F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301546dd55
+F ext/fts5/fts5_hash.c 3a82cb75caa64f6fc504ded02227d18938cc04d9ae4601836cc2e4b1d9f31055
 F ext/fts5/fts5_index.c 5fe14375a29e8a7aa8f3e863babe180a19269206c254c8f47b216821d4ac1e15
 F ext/fts5/fts5_main.c 24868f88ab2a865defbba7a92eebeb726cc991eb092b71b5f5508f180c72605b
 F ext/fts5/fts5_storage.c fb5ef3c27073f67ade2e1bea08405f9e43f68f5f3676ed0ab7013bce5ba10be6
@@ -126,7 +126,7 @@ F ext/fts5/fts5_vocab.c 1cd79854cb21543e66507b25b0578bc1b20aa6a1349b7feceb8e8fed
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
-F ext/fts5/test/fts5aa.test cba3fae6466446980caf1b9f5f26df77f95a999d35db7d932d6e82ae7ba0ede9
+F ext/fts5/test/fts5aa.test 7ef4b014578af5543231d8fb68e2c7e708f189a657ec3b780c813ec0f909f679
 F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18ca9385880bb7
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -321,6 +321,7 @@ F ext/rbu/rbuA.test 4e58e46e60d4064248614c43303d71f1b18cc804dd834ce6a913b3861828
 F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2
 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831
 F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead
+F ext/rbu/rbucollate.test 86d6fc9b8f59a27b7b5a6e20b5e29816d338a0dbdea8c54bfcc549a0d437f3ea
 F ext/rbu/rbucrash.test 61470d977a06a0abc2ec35b05d82a1d7d87d10f4ffabad14c1c231edc942ad66
 F ext/rbu/rbucrash2.test b2ecbdd7bb72c88bd217c65bd00dafa07f7f2d4d
 F ext/rbu/rbudiff.test 3e605cf624d00d04d0fb1316a3acec4fbe3b3ac5
@@ -337,9 +338,9 @@ F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
 F ext/rbu/rbutemplimit.test cd553a9288d515d0b5f87d277e76fd18c4aa740b761e7880fab11ce986ea18d1
 F ext/rbu/rbuvacuum.test ff357e9b556ca7ad4673da0ff7f244def919ff858e0f9f350d3e30fdd83a62a8
 F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa
-F ext/rbu/sqlite3rbu.c 64bd08c1011456f90564ed167abce3a9c2af421a924b21eb57231e078da04feb
+F ext/rbu/sqlite3rbu.c 4eea923892ab12d4dc34652bc2cab624053ec7aa234efbb8e3098dd7e7195c46
 F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2
-F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
+F ext/rbu/test_rbu.c baa23eb28457580673d2175e5f0c29ced0cd320ee819b13ad362398c53b96e90
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd
 F ext/repair/checkindex.c 7d28c01a2e012ac64257d230fc452b2cafb78311a91a343633d01d95220f66f3
@@ -350,7 +351,7 @@ F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc782
 F ext/repair/test/checkindex01.test 6945d0ffc0c1dc993b2ce88036b26e0f5d6fcc65da70fc9df27c2647bb358b0f
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c d941e44ad901da039caebb9f9fa99d81f2a4fc822e67cafe33fa4f6f789074a0
+F ext/rtree/rtree.c 88b43ba0455b888af37ee42b50785c5ea0e364b4a8efac3fa262bbf9b9747962
 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412
 F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436
 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2
@@ -488,7 +489,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
 F src/shell.c.in 4e1bcf8c70b8fb97c7cbaca6602e2a291d7fe17eff23a5de003d6fabd87f27d1
-F src/sqlite.h.in 959deaad89679e31d7f68fda668b0c5d1f592fffed7a9c1740fb8ded4e4e754a
+F src/sqlite.h.in c050d0749910f9ae101f126f5adbfe15c692e105a21ee5cec7e98a608d0cd847
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 99189e7611eb0bf98f21c7835dc74730a84e2e809c98e1e31c33896dee7a2849
 F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544
@@ -870,6 +871,7 @@ F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f
 F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb
 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
 F test/fts3corrupt3.test 56e0ee83e90b57f5f3644cb7d1b36a067b7b8b19cdf0dedce45e5e13cf752f65
+F test/fts3corrupt4.test 98022cbacbd6ddc4708f210768f5684f041f50ce330c461f2631752492611d96
 F test/fts3cov.test 9c3681325b9a850bca8dd75cc29dde73e9a87972bb75204e97d826f13c7181f9
 F test/fts3d.test d3e9c8fb75135ada06bf3bab4f9666224965d708
 F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963
@@ -877,7 +879,7 @@ F test/fts3defer2.test c540f5f5c2840f70c68fd9b597df817ec7170468
 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd
 F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297
 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
-F test/fts3expr.test 9466627007804d855bf9df2a0cfb3dac23686fdc
+F test/fts3expr.test ebae205a7a89446c32583bcd492dcb817b9f6b31819bb4dde2583bb99c77e526
 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
 F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8
 F test/fts3expr4.test c39a15d676b14fc439d9bf845aa7bddcf4a74dc3
@@ -1130,7 +1132,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 8ada8c1dee071e0fc275bc8bc2db7de537d625cad949d2200664b99a0a89eac5
+F test/permutations.test 528c92fbcbda44a723c354cbd2301ffd5586398be5ade7b360af3f035791cf3c
 F test/pragma.test 7c8cfc328a1717a95663cf8edb06c52ddfeaf97bb0aee69ae7457132e8d39e7d
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1700,10 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 395f8ea790e6e295800fa8927f0585b2419b9521ef4fd591d51d2a48db2a90c4
-R e7cccb4f976b9eb07c1586193cc26ec5
-T +bgcolor * #d0c0ff
-T +sym-release *
-T +sym-version-3.22.0 *
+P 68b898381ac2942965a3dbd416a45ddf813d6df7ea160f500ae4978e44a3a050
+R 875ae95303236d12f9a2fe1f9b260baa
 U drh
-Z 9c0e858ece53e1ee810a8a479ae4dc82
+Z 7916b9cfdd1d58ce6d6b2a3acbbaec70
@@ -1 +1 @@
-0c55d179733b46d8d0ba4d88e01a25e10677046ee3da1d5b1581e86726f2171d
+cbcbb1e5a98bde5a40e90faf8467b4f761d2ad7414cf4423571bdb8ba38a81d5
@@ -3566,6 +3566,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** </dl>
 */
 #define SQLITE_PREPARE_PERSISTENT              0x01
+#define SQLITE_PREPARE_NO_VTAB                 0x04

 /*
 ** CAPI3REF: Compiling An SQL Statement
@@ -4750,7 +4751,8 @@ int sqlite3_create_function_v2(
 ** to [sqlite3_create_function()], [sqlite3_create_function16()], or
 ** [sqlite3_create_function_v2()].
 */
-#define SQLITE_DETERMINISTIC    0x800
+#define SQLITE_DETERMINISTIC    0x000000800
+#define SQLITE_DIRECTONLY       0x000080000

 /*
 ** CAPI3REF: Deprecated Functions
@@ -0,0 +1,147 @@
+# 2006 September 9
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS3 module.
+#
+# $Id: fts3aa.test,v 1.1 2007/08/20 17:38:42 shess Exp $
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts3corrupt4
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  BEGIN;
+    CREATE VIRTUAL TABLE ft USING fts3;
+    INSERT INTO ft VALUES('aback');
+    INSERT INTO ft VALUES('abaft');
+    INSERT INTO ft VALUES('abandon');
+  COMMIT;
+}
+
+proc blob {a} { binary decode hex $a }
+db func blob blob
+
+do_execsql_test 1.1 {
+  SELECT quote(root) FROM ft_segdir;
+} {X'0005616261636B03010200030266740302020003046E646F6E03030200'}
+
+do_execsql_test 1.2 {
+  UPDATE ft_segdir SET root = blob(
+    '0005616261636B03010200 FFFFFFFF0702 66740302020003046E646F6E03030200'
+  );
+}
+
+do_catchsql_test 1.3 {
+  SELECT * FROM ft WHERE ft MATCH 'abandon';
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0.0 {
+  CREATE VIRTUAL TABLE ft USING fts3;
+  INSERT INTO ft(ft) VALUES('nodesize=32');
+}
+do_test 2.0.1 {
+  for {set i 0} {$i < 12} {incr i} {
+    execsql {
+      BEGIN;
+        INSERT INTO ft VALUES('abc' || $i);
+        INSERT INTO ft VALUES('abc' || $i || 'x' );
+        INSERT INTO ft VALUES('abc' || $i || 'xx' );
+      COMMIT
+    }
+  }
+  execsql {
+    SELECT count(*) FROM ft_segdir;
+    SELECT count(*) FROM ft_segments;
+  }
+} {12 0}
+
+do_execsql_test 2.1 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+  SELECT count(*) FROM ft_segdir;
+  SELECT count(*) FROM ft_segments;
+} {12 3}
+
+do_execsql_test 2.2 {
+  SELECT quote(block) FROM ft_segments WHERE blockid=2
+} {X'00056162633130031F0200'}
+
+db func blob blob
+do_execsql_test 2.3.1 {
+  UPDATE ft_segments SET block = 
+    blob('00056162633130031F0200 FFFFFFFF07FF55 66740302020003046E646F6E03030200')
+    WHERE blockid=2;
+} {}
+do_catchsql_test 2.3.2 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+} {1 {database disk image is malformed}}
+
+do_execsql_test 2.4.1 {
+  UPDATE ft_segments SET block = 
+    blob('00056162633130031F0200 02FFFFFFFF07 66740302020003046E646F6E03030200')
+    WHERE blockid=2;
+} {}
+do_catchsql_test 2.4.2 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+} {1 {database disk image is malformed}}
+
+do_execsql_test 2.5.1 {
+  UPDATE ft_segments SET block = 
+    blob('00056162633130031F0200 0202 6674 FFFFFF070302020003046E646F6E030200')
+    WHERE blockid=2;
+} {}
+do_catchsql_test 2.5.2 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0.0 {
+  CREATE VIRTUAL TABLE ft USING fts3;
+  INSERT INTO ft(ft) VALUES('nodesize=32');
+}
+do_test 3.0.1 {
+  execsql BEGIN
+  for {set i 0} {$i < 20} {incr i} {
+    execsql { INSERT INTO ft VALUES('abc' || $i) }
+  }
+  execsql {
+    COMMIT;
+    SELECT count(*) FROM ft_segdir;
+    SELECT count(*) FROM ft_segments;
+  }
+} {1 5}
+
+do_execsql_test 3.1 {
+  SELECT quote(root) FROM ft_segdir
+} {X'0101056162633132040136030132030136'}
+
+db func blob blob
+do_execsql_test 3.2 {
+  UPDATE ft_segdir 
+  SET root = blob('0101056162633132FFFFFFFF070236030132030136');
+}
+
+do_catchsql_test 3.1 {
+  SELECT * FROM ft WHERE ft MATCH 'abc20'
+} {1 {database disk image is malformed}}
+
+finish_test
+
+
@@ -409,7 +409,7 @@ do_test fts3expr-5.1 {
 } {1 {Usage: fts3_exprtest(tokenizer, expr, col1, ...}}
 do_test fts3expr-5.2 {
  catchsql { SELECT fts3_exprtest('doesnotexist', 'a b', 'c') }
-} {1 {No such tokenizer module}}
+} {1 {unknown tokenizer: doesnotexist}}
 do_test fts3expr-5.3 {
  catchsql { SELECT fts3_exprtest('simple', 'a b OR', 'c') }
 } {1 {Error parsing expression}}
@@ -255,6 +255,7 @@ test_suite "fts3" -prefix "" -description {
  fts3am.test fts3an.test fts3ao.test fts3atoken.test
  fts3auto.test fts3aux1.test fts3aux2.test fts3b.test
  fts3comp1.test fts3conf.test fts3corrupt2.test fts3corrupt.test
+  fts3corrupt4.test
  fts3cov.test fts3c.test fts3defer2.test fts3defer3.test
  fts3defer.test fts3drop.test fts3d.test fts3e.test
  fts3expr2.test fts3expr3.test fts3expr4.test fts3expr5.test
Author	SHA1	Message	Date
drh	bfcc1605d9	Import all FTS3/4 enhancements and fixes that exist on the latest trunk (3.31.0-beta) that do not require extensive change to the SQLite core into the 3.22 branch. Basically, the 3.31.0 FTS3 sources are copied into 3.22.0, with minor changes to work around core enhancements that are not available in 3.22.0. FossilOrigin-Name: cbcbb1e5a98bde5a40e90faf8467b4f761d2ad7414cf4423571bdb8ba38a81d5	2020-01-17 14:18:48 +00:00
drh	f11ec1b05c	Fix a buffer overread that could occur when running fts5 prefix queries inside a transaction. FossilOrigin-Name: 68b898381ac2942965a3dbd416a45ddf813d6df7ea160f500ae4978e44a3a050	2019-09-03 18:36:11 +00:00
drh	5dfdf191e9	Disable the undocumented rtreenode() SQL function that is only used for testing, except when doing a build that is specifically intended for testing. FossilOrigin-Name: 8452fd549966de12da35110bf4c87a2f34ade30ef44b6ed8252b43dc19fa830d	2019-09-03 18:04:15 +00:00
dan	79da32e190	Fix a problem with corrupt fts3 database detection introduced by [27199380]. FossilOrigin-Name: ceeb4fba84a651ddd432e87a968ec8d928030bf5b32c8790188d0fb5787c1b4b	2018-12-26 14:23:14 +00:00
dan	6b5bf1e039	Avoid a left-shift of a negative value (undefined behaviour) when dealing with a corrupt database in fts3. Cherrypick of [b851d12474]. FossilOrigin-Name: 2fa63a8be62a06136a34d32351229b54bea58253a63275edf82efe0e83d412ca	2018-12-26 11:39:39 +00:00
dan	2872b31f02	Change the way a comparison used to detect corrupt databases in fts3 is done to avoid potential pointer overflow in 32-bit builds. Cherrypick of [95a9a39ff7]. FossilOrigin-Name: 271993803cc99305642e9809cbe81cbefd944f4c42fc0de370e52eb69103dcdd	2018-12-24 13:39:13 +00:00
drh	7b7f7d915f	Add extra defenses against strategically corrupt databases to fts3/4. FossilOrigin-Name: c255889bd95bd5430dc7ced3317011ae2abb483d6c9af883af3dc7d6c2c2f234	2018-12-19 01:30:22 +00:00
dan	8991ef48cf	Speed up xAccess() calls made on an RBU VFS when there are lots of open connections. FossilOrigin-Name: fda8fdb0cbc3acf420613f5df4125898354184db52b8606dde55042688815ac7	2018-10-25 11:55:54 +00:00
dan	24ebe39365	Fix an RBU problem causing errors when updating tables with default collation sequences that require quoting (e.g. COLLATE "ICU_root-u-kn-on"). Cherrypick of [eb4f452e]. FossilOrigin-Name: 5dd61e1cbd11b375942baf72ed50ae9e55a09801e1a8c1cb679eaa9eaba4145c	2018-03-22 17:17:44 +00:00