Fix a buffer overread that could occur when running fts5 prefix queries inside a transaction.

FossilOrigin-Name: b54aa18b0fe4d683c602ed2ba59ded6c33168982d14ea14a12b4e00cde8bf973
Disable the undocumented rtreenode() SQL function that is only used for testing,
2019-09-03 19:40:52 +00:00 · 2019-09-03 17:46:35 +00:00 · 2018-12-19 01:38:18 +00:00 · 2017-07-21 07:56:09 +00:00 · 2017-06-17 09:59:36 +00:00 · 2017-06-17 00:39:47 +00:00
17 changed files with 270 additions and 57 deletions
@@ -1 +1 @@
-3.18.0
+3.18.2
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.18.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.18.2.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.18.0'
-PACKAGE_STRING='sqlite 3.18.0'
+PACKAGE_VERSION='3.18.2'
+PACKAGE_STRING='sqlite 3.18.2'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -1463,7 +1463,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.18.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.18.2 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1528,7 +1528,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.18.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.18.2:";;
   esac
  cat <<\_ACEOF

@@ -1652,7 +1652,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.18.0
+sqlite configure 3.18.2
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2071,7 +2071,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.18.0, which was
+It was created by sqlite $as_me 3.18.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -12151,7 +12151,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.18.0, which was
+This file was extended by sqlite $as_me 3.18.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12217,7 +12217,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.18.0
+sqlite config.status 3.18.2
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -1809,7 +1809,7 @@ static int fts3ScanInteriorNode(
  const char *zCsr = zNode;       /* Cursor to iterate through node */
  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
  char *zBuffer = 0;              /* Buffer to load terms into */
-  int nAlloc = 0;                 /* Size of allocated buffer */
+  i64 nAlloc = 0;                 /* Size of allocated buffer */
  int isFirstTerm = 1;            /* True when processing first term on page */
  sqlite3_int64 iChild;           /* Block id of child node to descend to */

@@ -1846,14 +1846,14 @@ static int fts3ScanInteriorNode(
    isFirstTerm = 0;
    zCsr += fts3GetVarint32(zCsr, &nSuffix);
    
-    if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
+    if( nPrefix<0 || nSuffix<0 || nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
-    if( nPrefix+nSuffix>nAlloc ){
+    if( (i64)nPrefix+nSuffix>nAlloc ){
      char *zNew;
-      nAlloc = (nPrefix+nSuffix) * 2;
-      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+      nAlloc = ((i64)nPrefix+nSuffix) * 2;
+      zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
      if( !zNew ){
        rc = SQLITE_NOMEM;
        goto finish_scan;
@@ -3344,7 +3344,7 @@ static int fts3ColumnMethod(
  }else if( iCol==p->nColumn ){
    /* The extra column whose name is the same as the table.
    ** Return a blob which is a pointer to the cursor.  */
-    sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
+    sqlite3_result_pointer(pCtx, pCsr);
  }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
    sqlite3_result_int64(pCtx, pCsr->iLangid);
  }else{
@@ -3559,16 +3559,13 @@ static int fts3FunctionArg(
  sqlite3_value *pVal,            /* argv[0] passed to function */
  Fts3Cursor **ppCsr              /* OUT: Store cursor handle here */
 ){
-  Fts3Cursor *pRet;
-  if( sqlite3_value_type(pVal)!=SQLITE_BLOB 
-   || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
-  ){
+  Fts3Cursor *pRet = (Fts3Cursor*)sqlite3_value_pointer(pVal);
+  if( pRet==0 ){
    char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
    sqlite3_result_error(pContext, zErr, -1);
    sqlite3_free(zErr);
    return SQLITE_ERROR;
  }
-  memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
  *ppCsr = pRet;
  return SQLITE_OK;
 }
@@ -1373,15 +1373,19 @@ static int fts3SegReaderNext(
  ** safe (no risk of overread) even if the node data is corrupted. */
  pNext += fts3GetVarint32(pNext, &nPrefix);
  pNext += fts3GetVarint32(pNext, &nSuffix);
-  if( nPrefix<0 || nSuffix<=0 
-   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
+  if( nSuffix<=0 
+   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
+   || nPrefix>pReader->nTermAlloc
  ){
    return FTS_CORRUPT_VTAB;
  }

-  if( nPrefix+nSuffix>pReader->nTermAlloc ){
-    int nNew = (nPrefix+nSuffix)*2;
-    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
+  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
+  ** overflow - hence the (i64) casts.  */
+  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
+    i64 nNew = ((i64)nPrefix+nSuffix)*2;
+    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
@@ -1403,7 +1407,7 @@ static int fts3SegReaderNext(
  ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  ** of these statements is untrue, then the data structure is corrupt.
  */
-  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
+  if( (&pReader->aNode[pReader->nNode] - pReader->aDoclist)<pReader->nDoclist
   || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  ){
    return FTS_CORRUPT_VTAB;
@@ -3726,6 +3730,9 @@ static int nodeReaderNext(NodeReader *p){
    }
    p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);

+    if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
+      return SQLITE_CORRUPT_VTAB;
+    }
    blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
    if( rc==SQLITE_OK ){
      memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
@@ -3733,6 +3740,9 @@ static int nodeReaderNext(NodeReader *p){
      p->iOff += nSuffix;
      if( p->iChild==0 ){
        p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
+        if( (p->nNode-p->iOff)<p->nDoclist ){
+          return SQLITE_CORRUPT_VTAB;
+        }
        p->aDoclist = &p->aNode[p->iOff];
        p->iOff += p->nDoclist;
      }
@@ -3740,7 +3750,6 @@ static int nodeReaderNext(NodeReader *p){
  }

  assert( p->iOff<=p->nNode );
-
  return rc;
 }

@@ -438,7 +438,9 @@ static int fts5HashEntrySort(
  for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
    Fts5HashEntry *pIter;
    for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
-      if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){
+      if( pTerm==0 
+       || (strlen(pIter->zKey)>=nTerm && 0==memcmp(pIter->zKey, pTerm, nTerm))
+      ){
        Fts5HashEntry *pEntry = pIter;
        pEntry->pScanNext = 0;
        for(i=0; ap[i]; i++){
@@ -561,6 +561,19 @@ do_test 20.1 {
  execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
 } $::ids

+#-------------------------------------------------------------------------
+do_execsql_test 25.0 {
+  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%);
+}
+do_execsql_test 25.1 {
+  BEGIN;
+  INSERT INTO t13 VALUES('AAAA');
+  SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB*');
+
+  END;
+}
+
+
 }


@@ -3520,6 +3520,7 @@ static int rtreeInit(
 }


+#if defined(SQLITE_TEST)
 /*
 ** Implementation of a scalar function that decodes r-tree nodes to
 ** human readable strings. This can be used for debugging and analysis.
@@ -3581,6 +3582,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){
  
  sqlite3_result_text(ctx, zText, -1, sqlite3_free);
 }
+#endif

 /* This routine implements an SQL function that returns the "depth" parameter
 ** from the front of a blob that is an r-tree node.  For example:
@@ -3610,9 +3612,11 @@ static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){
 */
 int sqlite3RtreeInit(sqlite3 *db){
  const int utf8 = SQLITE_UTF8;
-  int rc;
+  int rc = SQLITE_OK;

+#if defined(SQLITE_TEST)
  rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0);
+#endif
  if( rc==SQLITE_OK ){
    rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0);
  }
@@ -1,10 +1,10 @@
-C Add\sthe\snew\ssqlite3_set_last_insert_rowid()\sinterface\sto\sthe\sextension\nloader\sthunk.
-D 2017-03-25T19:16:41.259
+C Fix\sa\sbuffer\soverread\sthat\scould\soccur\swhen\srunning\sfts5\sprefix\squeries\sinside\sa\stransaction.
+D 2019-09-03T19:40:52.501
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a
 F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9
-F VERSION 3605fa447e4623f5ff4a6adc97b1fde9a257b8f2
+F VERSION 73dbb3aaa3d50b5bc513c1973949c879056115c8da845f413eefc0dc8f50141d
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
@@ -30,7 +30,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63
 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977
 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc
 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
-F configure b15d030a8ae2e672d76c97b6667dacb98934a467 x
+F configure 2128dbe97a28d3932f78d367abf63c82815504af4552b7f87690a5ae0dcd4f03 x
 F configure.ac 605173e829ab64514ed89f9b53d0da1739d7b0a0
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977
@@ -70,7 +70,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c f0d5de1bc2155ba7cd7c0c1a751779a3a8857fa34d5c12f3b233a23fa2e79ea2
+F ext/fts3/fts3.c 10aa75a09601f032ddda578cac06db49e53e21fee536afcca6ba3455153973a1
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a
 F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1
@@ -88,7 +88,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d
 F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c
-F ext/fts3/fts3_write.c a51d48d646974ee2fb4b17fcd5da0416a5759a32dcacc2cce2ba00d5a767848e
+F ext/fts3/fts3_write.c 0559fca37710bfeacc5c21f46f4ae66b31f6098560c39e686cd7d9f569c0798b
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
@@ -104,7 +104,7 @@ F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c c6ecc2280162a3714d15dce2a8f2299f748b627c
-F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2
+F ext/fts5/fts5_hash.c 2b1149273e77f0a700435307e9ee0bf830339316d0aa01dcdf8f81116d55ca3a
 F ext/fts5/fts5_index.c f67032a9a529ba52a545e6e3ab970764199c05d4
 F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2
 F ext/fts5/fts5_storage.c 8f0e65cb33bde8f449e1c9b4be4600d18b4da6e9
@@ -118,7 +118,7 @@ F ext/fts5/fts5_vocab.c e44fefa7f0c1db252998af071daf06a7147e17e7
 F ext/fts5/fts5parse.y e51b375403421b8b37428a89b095d00597129aae
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
-F ext/fts5/test/fts5aa.test bd2d88182b9f7f30d300044048ad14683306b745
+F ext/fts5/test/fts5aa.test 71f3ce62bce9b730b86e9ae9827a063ecf06f9b6b1e69beb6ab17f2da68f1d17
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
@@ -271,7 +271,7 @@ F ext/rbu/sqlite3rbu.c 2a89efba9eeba8e6c89a498dc195e8efbdde2694
 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba
 F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c 0acd285bfacc347579a5df9fe947212fb99e2775a40c43f027c3a16936c58e7e
+F ext/rtree/rtree.c ec7e8d76eeb29cb6d01767fb6569374b367f98aa2edb94136976f5964b81fec1
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
 F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec
 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba
@@ -344,7 +344,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c ae0e0397e6ad58465bbf932239ee7539ca22f257c97b16c9d0960a1f5de743a3
+F src/btree.c 8668d6eea7a12c455a1936fe1dbe9c5bb7fdb134578cddd5d0f77fda8d3bab4e
 F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d
 F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0
 F src/build.c 43f903c9082040ced2b421543cb0300c2973647d
@@ -403,7 +403,7 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa
 F src/shell.c ceb2b2f1f958ea2c47a7f37972d0f715fbf9dcf6a34a5e98c886b85e3ce6a238
-F src/sqlite.h.in 723107d97f2345a7c103632169dc61366121c4ab65d75a7d83c6dc0e5bbe5ca4
+F src/sqlite.h.in 28e36a38f5edecf54783faeac90d3aacf2c2f24e9945f9659a9395342604ceb0
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
 F src/sqliteInt.h a530e5baf5e559154d5c2e5cc57471bc780a7af4cd0a5d72750b1d850fef1e22
@@ -470,8 +470,8 @@ F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6
 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569
 F src/vdbe.c f1acf5744cef62cbfd0b503d84289f840b6cdc980ac47b0d9632dfdb89cc79eb
 F src/vdbe.h caa5346d52bae2a3c8c1dcfa60a7a4dc878a9e3865cb8239da55808b316c8158
-F src/vdbeInt.h 5db089ce18c4feff8820ec6e4cac2d2c82e03d4b1d96f10a6e43832147b8dffe
-F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860
+F src/vdbeInt.h b8770095a5226a25964c0804f39fd37b098845ffe49441c029445f4dc5c2d633
+F src/vdbeapi.c ef70ce2a72a72987902efef32ba2ed4d0ae90f161011c3d1547ad656a3f09422
 F src/vdbeaux.c ecd0468611925d218e1eb4b3f538907904b136f0e15e333291a232b521bfcef1
 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9
 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd
@@ -535,7 +535,7 @@ F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df
 F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972
 F test/autoindex4.test 49d3cd791a9baa16fb461d7ea3de80d019a819cf
 F test/autoindex5.test 96f084a5e6024ea07cace5888df3223f3ea86990
-F test/autovacuum.test 92c24eedbdb68e49f3fb71f26f9ce6d8988cac15
+F test/autovacuum.test 0831cd34e14695d297187f7f6519265e3121c5b0a1720e548e86829e796129e9
 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4
 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85
 F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d
@@ -775,6 +775,7 @@ F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c
 F test/fts3conf.test 60317efd562080e198b5bdc9fcd222ce32cf01d7
 F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb
 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
+F test/fts3corrupt4.test 98022cbacbd6ddc4708f210768f5684f041f50ce330c461f2631752492611d96
 F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
 F test/fts3d.test d3e9c8fb75135ada06bf3bab4f9666224965d708
 F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963
@@ -856,7 +857,7 @@ F test/in.test 20c5529986998949908f889c8208b2cd894b2cc9
 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75
 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0
 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068
-F test/in5.test 6c006e0bcd7351b69350ef566e65f244023489e9
+F test/in5.test 7ae37fcd4a5e198291c6ab5f31a5bb3d15397efe8b75a6736d7a95a7b8dd9e08
 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822
 F test/incrblob2.test a5ce5ed1d0b01e2ed347245a21170372528af0a5
 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4
@@ -1027,7 +1028,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test af720e7d139e7e5417341d0f0eef2b911c0b067852138dc2f5b6a451b5725118
+F test/permutations.test b174d61355dcf33f2c7c53a3e74559266a3288c41c1aa1fc52ec6a1f2825e67b
 F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1569,7 +1570,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 55df410b2cd27dc4c7670bcc1196fa3a0e9e601fc24c42e53d6b0a26ee3e4b45
-R e1b145e92e118b2bc2de7100fb7561b5
-U drh
-Z e401f4d2ca1590406e5405bb5cd2850e
+P 0a1cce496c515a2ff9c044021ac0e84756830f4ffbb86f5f736bdbb49fb74927
+Q +b3fa58dd7403dbd4d2e9f3ae23d7d1337830d6fef2aa2f137ac5174de0d5828e
+R b5d4ef6556ce4ce428174043997d4f82
+U dan
+Z 4eb9bb565f0c4ead36743645c9b39e85
@@ -1 +1 @@
-8469fc0d48d6af0accef9b8a84e08ad2ca32351907510d177b4ca4815c1ea7cb
+b54aa18b0fe4d683c602ed2ba59ded6c33168982d14ea14a12b4e00cde8bf973
@@ -8122,9 +8122,6 @@ int sqlite3BtreeInsert(
    ** btreeMoveto() call */
    if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){
      loc = 0;
-    }else if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0
-               && pCur->info.nKey==pX->nKey-1 ){
-      loc = -1;
    }else if( loc==0 ){
      rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc);
      if( rc ) return rc;
@@ -8175,12 +8172,18 @@ int sqlite3BtreeInsert(
      memcpy(newCell, oldCell, 4);
    }
    rc = clearCell(pPage, oldCell, &info);
-    if( info.nSize==szNew && info.nLocal==info.nPayload ){
+    if( info.nSize==szNew && info.nLocal==info.nPayload 
+     && (!ISAUTOVACUUM || szNew<pPage->minLocal)
+    ){
      /* Overwrite the old cell with the new if they are the same size.
      ** We could also try to do this if the old cell is smaller, then add
      ** the leftover space to the free list.  But experiments show that
      ** doing that is no faster then skipping this optimization and just
-      ** calling dropCell() and insertCell(). */
+      ** calling dropCell() and insertCell(). 
+      **
+      ** This optimization cannot be used on an autovacuum database if the
+      ** new entry uses overflow pages, as the insertCell() call below is
+      ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry.  */
      assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */
      if( oldCell+szNew > pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT;
      memcpy(oldCell, newCell, szNew);
@@ -4632,6 +4632,7 @@ int sqlite3_value_bytes16(sqlite3_value*);
 double sqlite3_value_double(sqlite3_value*);
 int sqlite3_value_int(sqlite3_value*);
 sqlite3_int64 sqlite3_value_int64(sqlite3_value*);
+void *sqlite3_value_pointer(sqlite3_value*);
 const unsigned char *sqlite3_value_text(sqlite3_value*);
 const void *sqlite3_value_text16(sqlite3_value*);
 const void *sqlite3_value_text16le(sqlite3_value*);
@@ -4945,6 +4946,7 @@ void sqlite3_result_error_code(sqlite3_context*, int);
 void sqlite3_result_int(sqlite3_context*, int);
 void sqlite3_result_int64(sqlite3_context*, sqlite3_int64);
 void sqlite3_result_null(sqlite3_context*);
+void sqlite3_result_pointer(sqlite3_context*, void*);
 void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*));
 void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64,
                           void(*)(void*), unsigned char encoding);
@@ -190,6 +190,7 @@ struct Mem {
    double r;           /* Real value used when MEM_Real is set in flags */
    i64 i;              /* Integer value used when MEM_Int is set in flags */
    int nZero;          /* Used when bit MEM_Zero is set in flags */
+    void *pPtr;         /* Pointer when flags==MEM_Ptr|MEM_Null */
    FuncDef *pDef;      /* Used only when flags==MEM_Agg */
    RowSet *pRowSet;    /* Used only when flags==MEM_RowSet */
    VdbeFrame *pFrame;  /* Used when flags==MEM_Frame */
@@ -247,6 +248,7 @@ struct Mem {
 ** policy for Mem.z.  The MEM_Term flag tells us whether or not the
 ** string is \000 or \u0000 terminated
 */
+#define MEM_Ptr       0x8000   /* u.pPtr is valid if type==SQLITE_NULL */
 #define MEM_Term      0x0200   /* String rep is nul terminated */
 #define MEM_Dyn       0x0400   /* Need to call Mem.xDel() on Mem.z */
 #define MEM_Static    0x0800   /* Mem.z points to a static string */
@@ -269,7 +271,7 @@ struct Mem {
 ** Clear any existing type flags from a Mem and replace them with f
 */
 #define MemSetTypeFlag(p, f) \
-   ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f)
+   ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero|MEM_Ptr))|f)

 /*
 ** Return true if a memory cell is not marked as invalid.  This macro
@@ -199,6 +199,11 @@ unsigned int sqlite3_value_subtype(sqlite3_value *pVal){
  Mem *pMem = (Mem*)pVal;
  return ((pMem->flags & MEM_Subtype) ? pMem->eSubtype : 0);
 }
+void *sqlite3_value_pointer(sqlite3_value *pVal){
+  Mem *p = (Mem*)pVal;
+  if( (p->flags&(MEM_TypeMask|MEM_Ptr))==(MEM_Null|MEM_Ptr) ) return p->u.pPtr;
+  return 0;
+}
 const unsigned char *sqlite3_value_text(sqlite3_value *pVal){
  return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8);
 }
@@ -373,6 +378,13 @@ void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){
  assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
  sqlite3VdbeMemSetInt64(pCtx->pOut, iVal);
 }
+void sqlite3_result_pointer(sqlite3_context *pCtx, void *pPtr){
+  assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
+  sqlite3VdbeMemSetNull(pCtx->pOut);
+  assert( (pCtx->pOut->flags & (MEM_TypeMask|MEM_Ptr))==MEM_Null );
+  pCtx->pOut->flags |= MEM_Ptr;
+  pCtx->pOut->u.pPtr = pPtr;
+}
 void sqlite3_result_null(sqlite3_context *pCtx){
  assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
  sqlite3VdbeMemSetNull(pCtx->pOut);
@@ -705,5 +705,12 @@ do_test autovacuum-9.5 {
  file size test.db
 } $::sqlite_pending_byte
 
+do_execsql_test autovacuum-10.1 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t1 VALUES(25, randomblob(104));
+  REPLACE INTO t1 VALUES(25, randomblob(1117));
+  PRAGMA integrity_check;
+} {ok}

 finish_test
@@ -0,0 +1,147 @@
+# 2006 September 9
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS3 module.
+#
+# $Id: fts3aa.test,v 1.1 2007/08/20 17:38:42 shess Exp $
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts3corrupt4
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  BEGIN;
+    CREATE VIRTUAL TABLE ft USING fts3;
+    INSERT INTO ft VALUES('aback');
+    INSERT INTO ft VALUES('abaft');
+    INSERT INTO ft VALUES('abandon');
+  COMMIT;
+}
+
+proc blob {a} { binary decode hex $a }
+db func blob blob
+
+do_execsql_test 1.1 {
+  SELECT quote(root) FROM ft_segdir;
+} {X'0005616261636B03010200030266740302020003046E646F6E03030200'}
+
+do_execsql_test 1.2 {
+  UPDATE ft_segdir SET root = blob(
+    '0005616261636B03010200 FFFFFFFF0702 66740302020003046E646F6E03030200'
+  );
+}
+
+do_catchsql_test 1.3 {
+  SELECT * FROM ft WHERE ft MATCH 'abandon';
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0.0 {
+  CREATE VIRTUAL TABLE ft USING fts3;
+  INSERT INTO ft(ft) VALUES('nodesize=32');
+}
+do_test 2.0.1 {
+  for {set i 0} {$i < 12} {incr i} {
+    execsql {
+      BEGIN;
+        INSERT INTO ft VALUES('abc' || $i);
+        INSERT INTO ft VALUES('abc' || $i || 'x' );
+        INSERT INTO ft VALUES('abc' || $i || 'xx' );
+      COMMIT
+    }
+  }
+  execsql {
+    SELECT count(*) FROM ft_segdir;
+    SELECT count(*) FROM ft_segments;
+  }
+} {12 0}
+
+do_execsql_test 2.1 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+  SELECT count(*) FROM ft_segdir;
+  SELECT count(*) FROM ft_segments;
+} {12 3}
+
+do_execsql_test 2.2 {
+  SELECT quote(block) FROM ft_segments WHERE blockid=2
+} {X'00056162633130031F0200'}
+
+db func blob blob
+do_execsql_test 2.3.1 {
+  UPDATE ft_segments SET block = 
+    blob('00056162633130031F0200 FFFFFFFF07FF55 66740302020003046E646F6E03030200')
+    WHERE blockid=2;
+} {}
+do_catchsql_test 2.3.2 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+} {1 {database disk image is malformed}}
+
+do_execsql_test 2.4.1 {
+  UPDATE ft_segments SET block = 
+    blob('00056162633130031F0200 02FFFFFFFF07 66740302020003046E646F6E03030200')
+    WHERE blockid=2;
+} {}
+do_catchsql_test 2.4.2 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+} {1 {database disk image is malformed}}
+
+do_execsql_test 2.5.1 {
+  UPDATE ft_segments SET block = 
+    blob('00056162633130031F0200 0202 6674 FFFFFF070302020003046E646F6E030200')
+    WHERE blockid=2;
+} {}
+do_catchsql_test 2.5.2 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0.0 {
+  CREATE VIRTUAL TABLE ft USING fts3;
+  INSERT INTO ft(ft) VALUES('nodesize=32');
+}
+do_test 3.0.1 {
+  execsql BEGIN
+  for {set i 0} {$i < 20} {incr i} {
+    execsql { INSERT INTO ft VALUES('abc' || $i) }
+  }
+  execsql {
+    COMMIT;
+    SELECT count(*) FROM ft_segdir;
+    SELECT count(*) FROM ft_segments;
+  }
+} {1 5}
+
+do_execsql_test 3.1 {
+  SELECT quote(root) FROM ft_segdir
+} {X'0101056162633132040136030132030136'}
+
+db func blob blob
+do_execsql_test 3.2 {
+  UPDATE ft_segdir 
+  SET root = blob('0101056162633132FFFFFFFF070236030132030136');
+}
+
+do_catchsql_test 3.1 {
+  SELECT * FROM ft WHERE ft MATCH 'abc20'
+} {1 {database disk image is malformed}}
+
+finish_test
+
+
@@ -237,4 +237,16 @@ do_execsql_test 8.4 {
  SELECT count(*) FROM n1 WHERE a IN (SELECT +a FROM n1)
 } 3

+#-------------------------------------------------------------------------
+# Test that ticket 61fe97454c is fixed.
+#
+do_execsql_test 9.0 {
+  CREATE TABLE t9(a INTEGER PRIMARY KEY);
+  INSERT INTO t9 VALUES (44), (45);
+}
+do_execsql_test 9.1 {
+  SELECT * FROM t9 WHERE a IN (44, 45, 44, 45)
+} {44 45}
+
+
 finish_test
@@ -253,6 +253,7 @@ test_suite "fts3" -prefix "" -description {
  fts3am.test fts3an.test fts3ao.test fts3atoken.test
  fts3auto.test fts3aux1.test fts3aux2.test fts3b.test
  fts3comp1.test fts3conf.test fts3corrupt2.test fts3corrupt.test
+  fts3corrupt4.test
  fts3cov.test fts3c.test fts3defer2.test fts3defer3.test
  fts3defer.test fts3drop.test fts3d.test fts3e.test
  fts3expr2.test fts3expr3.test fts3expr4.test fts3expr5.test
Author	SHA1	Message	Date
dan	5bb31f1319	Fix a buffer overread that could occur when running fts5 prefix queries inside a transaction. FossilOrigin-Name: b54aa18b0fe4d683c602ed2ba59ded6c33168982d14ea14a12b4e00cde8bf973	2019-09-03 19:40:52 +00:00
drh	66e41f3c1e	Disable the undocumented rtreenode() SQL function that is only used for testing, except when doing a build that is specifically intended for testing. FossilOrigin-Name: 0a1cce496c515a2ff9c044021ac0e84756830f4ffbb86f5f736bdbb49fb74927	2019-09-03 17:46:35 +00:00
drh	0e92e460f6	Add extra defenses against strategically corrupt databases to fts3/4. FossilOrigin-Name: 4bb21d8205b3c72b94442018a0544ecc55e3320ef2593f0e3350142b7f2a7663	2018-12-19 01:38:18 +00:00
drh	a1bea8fe53	Add new interfaces sqlite3_result_pointer(), and sqlite3_value_pointer() and use them to transfer the eponymous FTS3 column pointer to the snippet() and offsets() routines. This changes is a cherry-pick of [f0f49224] via [69906880cee] and is ultimately inspired by check-in [72de49f2]. FossilOrigin-Name: 8201f4e1c566f7223c71c07e6b703d1352801f1b2daa0fd00895a18e1944cb4d	2017-07-21 07:56:09 +00:00
drh	2e6df8c116	Version 3.18.2 FossilOrigin-Name: 036ebf729e4b21035d7f4f8e35a6f705e6bf99887889e2dc14ebf2242e7930dd	2017-06-17 09:59:36 +00:00
drh	83b7d0143a	When building an ephemeral table of integers, do not assume that the table does not already contain N if the insert cursor points to N-1. Fix for ticket [61fe9745]. Also increase the version number to 3.18.2. This is the release candidate. FossilOrigin-Name: e1b71029087675749a5c64303a8756d16cb33ed3ac2e8d3b075051f575c072fb	2017-06-17 00:39:47 +00:00
drh	86f5c3fdff	Version 3.18.1 FossilOrigin-Name: 77bb46233db03a3338bacf7e56f439be3dfd1926ea0c44d252eeafa7a7b31c06	2017-06-16 13:41:15 +00:00
drh	004ba54048	An unintentional fork was created due to a typo when creating branch-3.18. This check-in resolves the fork and is the 3.18.1 release candidate. FossilOrigin-Name: 20e5ffb1ceed514bf812ea5a2ce027b55203612bdbf8116a69f0cd52c67dbb84	2017-06-16 12:15:25 +00:00
drh	5da1cd0554	Increase the version number to 3.18.1. FossilOrigin-Name: 4be2eba8c15104e9f420738092c06a14a9c9c34b13c0c8f897302ab7613ffccc	2017-06-15 07:47:09 +00:00
drh	de415c006a	Ensure pointer map entries are always added when a row that does use overflow pages replaces one that does not in an auto-vacuum database. Fix for [fda22108]. FossilOrigin-Name: 30c50f0e06084f3c4357962ed99e2c803b21ca238d00d70d9dee40164caf2afa	2017-06-12 23:44:37 +00:00
drh	fe329a7370	Version 3.18.0 Release Candidate FossilOrigin-Name: 424a0d380332858ee55bdebc4af3789f74e70a2b3ba1cf29d84b9b4bcf3e2e37	2017-03-28 18:48:43 +00:00
@@ -1 +1 @@
 .18.0
 .18.2