Fix problems in the previous commit.

FossilOrigin-Name: 74ad80eb745c708bdbbbf24cbbbf50971e84182f
Add extra (somewhat inefficient) trace callbacks for triggers if SQLITE_TRACE_TRIGGER is defined.
2017-01-21 16:21:24 +00:00 · 2017-01-21 15:58:42 +00:00 · 2017-01-21 15:55:41 +00:00 · 2017-01-21 14:11:28 +00:00 · 2017-01-20 20:43:14 +00:00 · 2017-01-20 16:47:34 +00:00
22 changed files with 478 additions and 123 deletions
@@ -95,6 +95,7 @@ static int vtshimCreate(
  if( rc ){
    sqlite3_free(pNew);
    *ppVtab = 0;
+    return rc;
  }
  pNew->pAux = pAux;
  pNew->ppPrev = &pAux->pAllVtab;
@@ -133,6 +134,7 @@ static int vtshimConnect(
  if( rc ){
    sqlite3_free(pNew);
    *ppVtab = 0;
+    return rc;
  }
  pNew->pAux = pAux;
  pNew->ppPrev = &pAux->pAllVtab;
@@ -0,0 +1,66 @@
+# 2014 October 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set ::testprefix rbufault4
+
+for {set tn 1} {1} {incr tn} {
+  reset_db
+  do_execsql_test 1.0 {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
+    CREATE INDEX i1b ON t1(b);
+    CREATE INDEX i1c ON t1(c);
+    INSERT INTO t1 VALUES(1, 2, 3);
+    INSERT INTO t1 VALUES(4, 5, 6);
+  }
+
+  forcedelete test.db2
+  sqlite3rbu_vacuum rbu test.db test.db2
+  for {set i 0} {$i < $tn} {incr i} { rbu step }
+  set rc [rbu close]
+  if {$rc!="SQLITE_OK"} { 
+    if {$rc!="SQLITE_DONE"} {error $rc}
+    break
+  }
+  faultsim_save
+
+  do_faultsim_test $tn -faults oom-t* -prep {
+    faultsim_restore
+  } -body {
+    sqlite3rbu_vacuum rbu test.db test.db2
+    while 1 {
+      set rc [rbu step]
+      if {$rc=="SQLITE_DONE"} break
+      if {$rc!="SQLITE_OK"} { error $rc }
+    }
+  } -test {
+    catch {rbu close}
+    faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM}
+
+    sqlite3rbu_vacuum rbu test.db test.db2
+    while {[rbu step]=="SQLITE_OK"} {}
+    set trc [rbu close]
+    if {$trc!="SQLITE_DONE"} { error "Got $trc instead of SQLITE_DONE!" }
+
+    set rc [db one {PRAGMA integrity_check}]
+    if {$rc!="ok"} { error "Got $rc instead of ok!" }
+  }
+}
+
+
+
+finish_test
+
@@ -199,6 +199,37 @@ if {$::tcl_platform(platform)=="unix"} {
  }
 }

+#-------------------------------------------------------------------------
+# Test the outcome of some other connection running a checkpoint while
+# the incremental checkpoint is suspended.
+#
+reset_db
+do_execsql_test 6.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
+  CREATE INDEX i1b ON t1(b);
+  CREATE INDEX i1c ON t1(c);
+  INSERT INTO t1 VALUES(1, 2, 3);
+  INSERT INTO t1 VALUES(4, 5, 6);
+}
+forcedelete test.db2
+
+do_test 6.1 {
+  sqlite3rbu_vacuum rbu test.db test.db2
+  while {[rbu state]!="checkpoint"} { rbu step }
+  rbu close
+} {SQLITE_OK}
+
+do_execsql_test 6.2 {
+  SELECT 1 FROM sqlite_master LIMIT 1;
+  PRAGMA wal_checkpoint;
+} {1 0 4 4}
+
+do_test 6.3 {
+  sqlite3rbu_vacuum rbu test.db test.db2
+  while {[rbu step]!="SQLITE_DONE"} { rbu step }
+  rbu close
+  execsql { PRAGMA integrity_check }
+} {ok}

 finish_test

@@ -2333,7 +2333,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){
 ** Open the database handle and attach the RBU database as "rbu". If an
 ** error occurs, leave an error code and message in the RBU handle.
 */
-static void rbuOpenDatabase(sqlite3rbu *p){
+static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){
  assert( p->rc || (p->dbMain==0 && p->dbRbu==0) );
  assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 );

@@ -2420,6 +2420,15 @@ static void rbuOpenDatabase(sqlite3rbu *p){
    if( !rbuIsVacuum(p) ){
      p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
    }else if( p->pRbuFd->pWalFd ){
+      if( pbRetry ){
+        p->pRbuFd->bNolock = 0;
+        sqlite3_close(p->dbRbu);
+        sqlite3_close(p->dbMain);
+        p->dbMain = 0;
+        p->dbRbu = 0;
+        *pbRetry = 1;
+        return;
+      }
      p->rc = SQLITE_ERROR;
      p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database");
    }else{
@@ -2600,16 +2609,18 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){
    if( rc2!=SQLITE_INTERNAL ) p->rc = rc2;
  }

-  if( p->rc==SQLITE_OK ){
+  if( p->rc==SQLITE_OK && p->nFrame>0 ){
    p->eStage = RBU_STAGE_CKPT;
    p->nStep = (pState ? pState->nRow : 0);
    p->aBuf = rbuMalloc(p, p->pgsz);
    p->iWalCksum = rbuShmChecksum(p);
  }

-  if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){
-    p->rc = SQLITE_DONE;
-    p->eStage = RBU_STAGE_DONE;
+  if( p->rc==SQLITE_OK ){
+    if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){
+      p->rc = SQLITE_DONE;
+      p->eStage = RBU_STAGE_DONE;
+    }
  }
 }

@@ -2782,7 +2793,7 @@ static void rbuMoveOalFile(sqlite3rbu *p){
 #endif

      if( p->rc==SQLITE_OK ){
-        rbuOpenDatabase(p);
+        rbuOpenDatabase(p, 0);
        rbuSetupCheckpoint(p, 0);
      }
    }
@@ -3493,6 +3504,7 @@ static sqlite3rbu *openRbuHandle(
    /* Open the target, RBU and state databases */
    if( p->rc==SQLITE_OK ){
      char *pCsr = (char*)&p[1];
+      int bRetry = 0;
      if( zTarget ){
        p->zTarget = pCsr;
        memcpy(p->zTarget, zTarget, nTarget+1);
@@ -3504,7 +3516,18 @@ static sqlite3rbu *openRbuHandle(
      if( zState ){
        p->zState = rbuMPrintf(p, "%s", zState);
      }
-      rbuOpenDatabase(p);
+
+      /* If the first attempt to open the database file fails and the bRetry
+      ** flag it set, this means that the db was not opened because it seemed
+      ** to be a wal-mode db. But, this may have happened due to an earlier
+      ** RBU vacuum operation leaving an old wal file in the directory.
+      ** If this is the case, it will have been checkpointed and deleted
+      ** when the handle was closed and a second attempt to open the 
+      ** database may succeed.  */
+      rbuOpenDatabase(p, &bRetry);
+      if( bRetry ){
+        rbuOpenDatabase(p, 0);
+      }
    }

    if( p->rc==SQLITE_OK ){
@@ -893,7 +893,7 @@ wordcount$(EXE):	$(TOP)/test/wordcount.c sqlite3.c
 		$(TOP)/test/wordcount.c sqlite3.c

 speedtest1$(EXE):	$(TOP)/test/speedtest1.c sqlite3.o
-	$(TCC) -I. $(OTAFLAGS) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB) 
+	$(TCCX) -I. $(OTAFLAGS) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB) 

 rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o 
 	$(TCC) -I. -o rbu$(EXE) $(TOP)/ext/rbu/rbu.c sqlite3.o \
@@ -1,5 +1,5 @@
-C Back\sout\scheck-in\s[0b3174e0b1364c]\sand\sreplace\sit\swith\sa\sbetter\sfix\nfor\sticket\s[91e2e8ba6ff2e2]\s-\sa\sfix\sthat\sdoes\snot\scause\sthe\sproblem\nidentified\sby\sticket\s[7ffd1ca1d2ad4ec].
-D 2017-01-16T16:43:02.076
+C Fix\sproblems\sin\sthe\sprevious\scommit.
+D 2017-01-21T16:21:24.115
 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da
@@ -227,7 +227,7 @@ F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
 F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
-F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e
+F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
 F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
@@ -252,13 +252,14 @@ F ext/rbu/rbudor.test 99b05cc0df613e962c2c8085cfb05686a09cf315
 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89
 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca
+F ext/rbu/rbufault4.test 34e70701cbec51571ffbd9fbf9d4e0f2ec495ca7
 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda
 F ext/rbu/rbuprogress.test e3e25fb7622641b8f2df7c6b7a7eb6fddfc46a4b
 F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83
 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
 F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0
-F ext/rbu/rbuvacuum2.test 2569205b74ff40fbf3bda2fce33a58eb40eebdcc
-F ext/rbu/sqlite3rbu.c 4ca89fbe9ca501da17f338d580d3f19d59fc6053
+F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa
+F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318
 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba
 F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
@@ -313,7 +314,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 2cf5f0362c5687fd8e912c3a327b49a2e8ba0f9b
+F main.mk 8f620f3418d7252a3f428f5aacf674b5507807a8
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -332,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 69966fb2c574954cd3216f09407c9a02c52d3bd7
+F src/btree.c e6b8f39d7dd1ad65f1747956de36f2595172a23d
 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac
 F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e
 F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af
@@ -353,7 +354,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 05e47e2de7b712a3a4148cd469e5f60873f5ef13
 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e
-F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d
+F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258
 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0
 F src/malloc.c fc1b9f445290f2145da48fc08730c26e6082b640
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -387,17 +388,17 @@ F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a
 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a
 F src/printf.c ff10a9b9902cd2afe5f655f3013c6307d969b1fd
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb
+F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6
-F src/shell.c 6095531aa900decdaa765e0f3993fba7153c92c1
+F src/shell.c 0344f71dc008df2433ae10175f59ad432c428810
 F src/sqlite.h.in e71655293c9bde26939496f3aac9d1821d2c07a2
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae
-F src/sqliteInt.h bec6274d8991528bc12d9a34d01fe84bdf6d00d9
+F src/sqliteInt.h 525c061ae9aafc8d4720a018d82f0936d9eee5ab
 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
-F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9
+F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 418f5e5e0840425a7e5b33f3600dccd378a57549
 F src/test1.c 8a98191a1da8e100f77cdb5cc716df67d405028d
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
@@ -445,8 +446,8 @@ F src/test_tclvar.c df9fe1213c2634687a9ca0b0bec0d2119d359ae3
 F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858
 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
-F src/test_windirent.c 600398db0198ca1c77ca183831bf456746b6f5c4
-F src/test_windirent.h 7edc57e2faa727026dbd5d010dd0e2e665d5aa01
+F src/test_windirent.c 17f91f5f2aa1bb7328abb49414c363b5d2a9d3ff
+F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5
@@ -456,12 +457,12 @@ F src/update.c b356b29d04c71f33c779f2cb557cf953819bdd7a
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c a88b0466fddf445ce752226d4698ca3faada620a
 F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16
-F src/vdbe.c c7add5978cb84ae3a7bcb16f8b56cb3bbdf04b7e
+F src/vdbe.c 132f552033f72820c27a100861fa83254358cbcd
 F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b
-F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e
+F src/vdbeInt.h 4cad10b68ed052104a997454540ea71ceb2888e1
 F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24
 F src/vdbeaux.c 35c9a9908174e5a26c96d15e1f98214814a39147
-F src/vdbeblob.c f4f98ea672b242f807c08c92c7faaa79e5091b65
+F src/vdbeblob.c 824f360105b8cd43c2ec8c4611fd3b162a3a3eed
 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd
 F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face
 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834
@@ -897,7 +898,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac
 F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a
 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0
 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff
-F test/kvtest.c 2c66ddefcd03c2caa337f6dd79e6c82368af83df
+F test/kvtest.c 9e428931f0733b5ba65b6b5abd95b27320e875a3
 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63
 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200
 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302
@@ -1130,7 +1131,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/speedtest1.c 4e8ea6165046f02e1cfe0f4700256e91c981ec10
+F test/speedtest1.c 02fe15bb784c5276a083ffe9969cc51e0bce7644
 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
@@ -1382,7 +1383,7 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292
 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96
 F test/vtabE.test d5024aa42754962f6bb0afd261681686488e7afe
 F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b
-F test/vtabH.test 97f61b0253260831af6232163f7852e2653baed6
+F test/vtabH.test a2912cd3ea3386aecc3cb74ebfdfcc4e6d4b7dd3
 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f
 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
@@ -1506,7 +1507,7 @@ F tool/showstat4.c b14159aa062f661b394ba37b6b7b94bfb8012ab9
 F tool/showwal.c ec79959834f7b21f1e0a2aa52bb7c056d2203977
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/spaceanal.tcl ab7d9bf68062907282a64b3e12ccbfad47193c5a
-F tool/speed-check.sh e6ca0695b047af64201ebe0ef452e423f55d78b1
+F tool/speed-check.sh 65ac2f5b00771b9dcefb95bebae1aab76c537ea3
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
@@ -1546,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9d0dfe0b088a5917afa06207ca3ac5618e3da82f
-R d9bec5fa752ad661e71dee0a4ee8e911
-U drh
-Z 5e9d81347f66aedf2ffc7c8c671d9f61
+P ffda1d1e1c858abd02f0c07b906cfac5ad075498
+R c37bc6770e2789d7f8851d61fd232055
+U dan
+Z a07b3b8a92ddc391359da9de406f8455
@@ -1 +1 @@
-0613665274346917f5482f9210bf0c60a0fed7d9
+74ad80eb745c708bdbbbf24cbbbf50971e84182f
@@ -4523,16 +4523,14 @@ static int accessPayload(
            pCur->aOverflow, nOvfl*2*sizeof(Pgno)
        );
        if( aNew==0 ){
-          rc = SQLITE_NOMEM_BKPT;
+          return SQLITE_NOMEM_BKPT;
        }else{
          pCur->nOvflAlloc = nOvfl*2;
          pCur->aOverflow = aNew;
        }
      }
-      if( rc==SQLITE_OK ){
-        memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno));
-        pCur->curFlags |= BTCF_ValidOvfl;
-      }
+      memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno));
+      pCur->curFlags |= BTCF_ValidOvfl;
    }

    /* If the overflow page-list cache has been allocated and the
@@ -4547,8 +4545,8 @@ static int accessPayload(
      offset = (offset%ovflSize);
    }

-    for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){
-
+    assert( rc==SQLITE_OK && amt>0 );
+    while( nextPage ){
      /* If required, populate the overflow page-list cache. */
      if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){
        assert( pCur->aOverflow[iIdx]==0
@@ -4637,6 +4635,9 @@ static int accessPayload(
        amt -= a;
        pBuf += a;
      }
+      if( amt==0 ) break;
+      if( rc ) break;
+      iIdx++;
    }
  }

@@ -18,7 +18,6 @@
 #endif
 #include "sqlite3ext.h"
 #include "sqliteInt.h"
-#include <string.h>

 #ifndef SQLITE_OMIT_LOAD_EXTENSION
 /*
@@ -15,8 +15,6 @@
 ** table and column.
 */
 #include "sqliteInt.h"
-#include <stdlib.h>
-#include <string.h>

 /*
 ** Walk the expression tree pExpr and increase the aggregate function
@@ -614,6 +614,7 @@ struct ShellState {
  int autoExplain;       /* Automatically turn on .explain mode */
  int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
  int statsOn;           /* True to display memory stats before each finalize */
+  int vmstepsOn;         /* Display VM steps before each finalize */
  int scanstatsOn;       /* True to display scan stats before each finalize */
  int countChanges;      /* True to display change counts */
  int backslashOn;       /* Resolve C-style \x escapes in SQL input text */
@@ -1997,6 +1998,12 @@ static int shell_exec(
        display_stats(db, pArg, 0);
      }

+      if( pArg && pArg->vmstepsOn && pStmt ){
+        int iCur = sqlite3_stmt_status(pStmt, SQLITE_STMTSTATUS_VM_STEP, 0);
+        FILE *out = pArg->traceOut ? pArg->traceOut : pArg->out;
+        raw_printf(out, "VM steps: %d\n", iCur);
+      }
+
      /* print loop-counters if required */
      if( pArg && pArg->scanstatsOn ){
        display_scanstats(db, pArg);
@@ -5339,6 +5346,16 @@ static int do_meta_command(char *zLine, ShellState *p){
    }
  }else

+  if( c=='v' && strncmp(azArg[0], "vmsteps", n)==0 ){
+    if( nArg==2 ){
+      p->vmstepsOn = booleanValue(azArg[1]);
+    }else{
+      raw_printf(stderr, "Usage: .vmsteps ?on|off?\n");
+      rc = 1;
+    }
+  }else
+
+
 #if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE)
  if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){
    sqlite3WhereTrace = nArg>=2 ? booleanValue(azArg[1]) : 0xff;
@@ -111,7 +111,7 @@
 #endif

 /* What version of CLANG is being used.  0 means CLANG is not being used */
-#ifdef __clang__
+#if defined(__clang__) && !defined(_WIN32)
 # define CLANG_VERSION \
            (__clang_major__*1000000+__clang_minor__*1000+__clang_patchlevel__)
 #else
@@ -511,6 +511,18 @@
 #include <assert.h>
 #include <stddef.h>

+/*
+** Use a macro to replace memcpy() if compiled with SQLITE_INLINE_MEMCPY.
+** This allows better measurements of where memcpy() is used when running
+** cachegrind.  But this macro version of memcpy() is very slow so it
+** should not be used in production.  This is a performance measurement
+** hack only.
+*/
+#ifdef SQLITE_INLINE_MEMCPY
+# define memcpy(D,S,N) {char*xxd=(char*)(D);const char*xxs=(const char*)(S);\
+                        int xxn=(N);while(xxn-->0)*(xxd++)=*(xxs++);}
+#endif
+
 /*
 ** If compiling for a processor that lacks floating point support,
 ** substitute integer for floating-point
@@ -17,8 +17,6 @@
 ** if they are not used.
 */
 #include "sqliteInt.h"
-#include <stdlib.h>
-#include <string.h>

 #ifndef SQLITE_OMIT_GET_TABLE

@@ -63,6 +63,7 @@ LPDIR opendir(
    dirname = windirent_getenv("SystemDrive");
  }

+  memset(&data, 0, sizeof(struct _finddata_t));
  _snprintf(data.name, namesize, "%s\\*", dirname);
  dirp->d_handle = _findfirst(data.name, &data);

@@ -71,12 +72,18 @@ LPDIR opendir(
    return NULL;
  }

-  /* TODO: Remove this block to allow hidden and system files. */
-  if( data.attrib&_A_HIDDEN || data.attrib&_A_SYSTEM ){
+  /* TODO: Remove this block to allow hidden and/or system files. */
+  if( is_filtered(data) ){
+next:
+
+    memset(&data, 0, sizeof(struct _finddata_t));
    if( _findnext(dirp->d_handle, &data)==-1 ){
      closedir(dirp);
      return NULL;
    }
+
+    /* TODO: Remove this block to allow hidden and/or system files. */
+    if( is_filtered(data) ) goto next;
  }

  dirp->d_first.d_attributes = data.attrib;
@@ -105,11 +112,11 @@ LPDIRENT readdir(

 next:

+  memset(&data, 0, sizeof(struct _finddata_t));
  if( _findnext(dirp->d_handle, &data)==-1 ) return NULL;

-  /* TODO: Remove this block to allow hidden and system files. */
-  if( data.attrib&_A_HIDDEN ) goto next;
-  if( data.attrib&_A_SYSTEM ) goto next;
+  /* TODO: Remove this block to allow hidden and/or system files. */
+  if( is_filtered(data) ) goto next;

  dirp->d_next.d_ino++;
  dirp->d_next.d_attributes = data.attrib;
@@ -146,14 +153,14 @@ INT readdir_r(

 next:

+  memset(&data, 0, sizeof(struct _finddata_t));
  if( _findnext(dirp->d_handle, &data)==-1 ){
    *result = NULL;
    return ENOENT;
  }

-  /* TODO: Remove this block to allow hidden and system files. */
-  if( data.attrib&_A_HIDDEN ) goto next;
-  if( data.attrib&_A_SYSTEM ) goto next;
+  /* TODO: Remove this block to allow hidden and/or system files. */
+  if( is_filtered(data) ) goto next;

  entry->d_ino = (ino_t)-1; /* not available */
  entry->d_attributes = data.attrib;
@@ -92,6 +92,17 @@ struct DIR {
  DIRENT d_next;     /* DIRENT constructed based on "_findnext". */
 };

+/*
+** Provide a macro, for use by the implementation, to determine if a
+** particular directory entry should be skipped over when searching for
+** the next directory entry that should be returned by the readdir() or
+** readdir_r() functions.
+*/
+
+#ifndef is_filtered
+#  define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM))
+#endif
+
 /*
 ** Provide the function prototype for the POSIX compatiable getenv()
 ** function.  This function is not thread-safe.
@@ -960,6 +960,25 @@ case OP_Halt: {
    p->nFrame--;
    sqlite3VdbeSetChanges(db, p->nChange);
    pcx = sqlite3VdbeFrameRestore(pFrame);
+#ifdef SQLITE_TRACE_TRIGGER
+    if( (db->mTrace & SQLITE_TRACE_STMT) && aOp[0].p4.z ){
+      int nTotal = nVmStep - pFrame->nVmStep;
+      char *zTrace;
+      assert( nTotal>=0 );
+      assert( db->xTrace );
+      zTrace = sqlite3_mprintf("%.*s%s completed (VM steps: total=%d self=%d)", 
+          p->nFrame, "                                                    :",
+          aOp[0].p4.z, nTotal, nVmStep - pFrame->nVmStepAdj
+      );
+      if( zTrace ){
+        (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, 0, zTrace);
+        sqlite3_free(zTrace);
+      }
+      if( p->pFrame ){
+        p->pFrame->nVmStepAdj += nTotal;
+      }
+    }
+#endif
    lastRowid = db->lastRowid;
    if( pOp->p2==OE_Ignore ){
      /* Instruction pcx is the OP_Program that invoked the sub-program 
@@ -5854,6 +5873,10 @@ case OP_Program: {        /* jump */
  pFrame->nDbChange = p->db->nChange;
  assert( pFrame->pAuxData==0 );
  pFrame->pAuxData = p->pAuxData;
+#ifdef SQLITE_TRACE_TRIGGER
+  pFrame->nVmStep = nVmStep;
+  pFrame->nVmStepAdj = nVmStep;
+#endif
  p->pAuxData = 0;
  p->nChange = 0;
  p->pFrame = pFrame;
@@ -6889,6 +6912,17 @@ case OP_Init: {          /* jump */
      x(db->pTraceArg, z);
      sqlite3_free(z);
    }else
+#endif
+#ifdef SQLITE_TRACE_TRIGGER
+    if( p->pFrame ){
+      zTrace = sqlite3_mprintf("%.*s%s", p->nFrame-1, 
+          "                                                    :", zTrace
+      );
+      if( zTrace ){
+        (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, zTrace);
+        sqlite3_free(zTrace);
+      }
+    }else
 #endif
    {
      (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, zTrace);
@@ -175,6 +175,10 @@ struct VdbeFrame {
  int nChildCsr;          /* Number of cursors for child frame */
  int nChange;            /* Statement changes (Vdbe.nChange)     */
  int nDbChange;          /* Value of db->nChange */
+#ifdef SQLITE_TRACE_TRIGGER
+  int nVmStep;            /* Value of nVmStep at start of program */
+  int nVmStepAdj;         /* Adjusted for nested programs */
+#endif
 };

 #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))])
@@ -23,10 +23,10 @@
 */
 typedef struct Incrblob Incrblob;
 struct Incrblob {
-  int flags;              /* Copy of "flags" passed to sqlite3_blob_open() */
  int nByte;              /* Size of open blob, in bytes */
  int iOffset;            /* Byte offset of blob in cursor data */
-  int iCol;               /* Table column this handle is open on */
+  u16 iCol;               /* Table column this handle is open on */
+  u8 isPureKV;            /* True if pTab is a pure key/value table */
  BtCursor *pCsr;         /* Cursor pointing at blob row */
  sqlite3_stmt *pStmt;    /* Statement holding cursor open */
  sqlite3 *db;            /* The associated database */
@@ -57,12 +57,17 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){
  char *zErr = 0;                 /* Error message */
  Vdbe *v = (Vdbe *)p->pStmt;

-  /* Set the value of the SQL statements only variable to integer iRow. 
-  ** This is done directly instead of using sqlite3_bind_int64() to avoid 
-  ** triggering asserts related to mutexes.
+  /* Set the value of register r[1] in the SQL statement to integer iRow. 
+  ** This is done directly as a performance optimization
  */
-  assert( v->aVar[0].flags&MEM_Int );
-  v->aVar[0].u.i = iRow;
+  v->aMem[1].flags = MEM_Int;
+  v->aMem[1].u.i = iRow;
+
+  /* If the statement has been run before (and is paused at the OP_ResultRow)
+  ** then back it up to the point where it does the OP_SeekRowid.  This could
+  ** have been down with an extra OP_Goto, but simply setting the program
+  ** counter is faster. */
+  if( v->pc>3 ) v->pc = 3;

  rc = sqlite3_step(p->pStmt);
  if( rc==SQLITE_ROW ){
@@ -112,7 +117,7 @@ int sqlite3_blob_open(
  const char *zTable,     /* The table containing the blob */
  const char *zColumn,    /* The column containing the blob */
  sqlite_int64 iRow,      /* The row containing the glob */
-  int flags,              /* True -> read/write access, false -> read-only */
+  int wrFlag,             /* True -> read/write access, false -> read-only */
  sqlite3_blob **ppBlob   /* Handle for accessing the blob returned here */
 ){
  int nAttempt = 0;
@@ -134,7 +139,7 @@ int sqlite3_blob_open(
    return SQLITE_MISUSE_BKPT;
  }
 #endif
-  flags = !!flags;                /* flags = (flags ? 1 : 0); */
+  wrFlag = !!wrFlag;                /* wrFlag = (wrFlag ? 1 : 0); */

  sqlite3_mutex_enter(db->mutex);

@@ -194,9 +199,8 @@ int sqlite3_blob_open(

    /* If the value is being opened for writing, check that the
    ** column is not indexed, and that it is not part of a foreign key. 
-    ** It is against the rules to open a column to which either of these
-    ** descriptions applies for writing.  */
-    if( flags ){
+    */
+    if( wrFlag ){
      const char *zFault = 0;
      Index *pIdx;
 #ifndef SQLITE_OMIT_FOREIGN_KEY
@@ -257,18 +261,17 @@ int sqlite3_blob_open(
      static const VdbeOpList openBlob[] = {
        {OP_TableLock,      0, 0, 0},  /* 0: Acquire a read or write lock */
        {OP_OpenRead,       0, 0, 0},  /* 1: Open a cursor */
-        {OP_Variable,       1, 1, 0},  /* 2: Move ?1 into reg[1] */
-        {OP_NotExists,      0, 7, 1},  /* 3: Seek the cursor */
-        {OP_Column,         0, 0, 1},  /* 4  */
-        {OP_ResultRow,      1, 0, 0},  /* 5  */
-        {OP_Goto,           0, 2, 0},  /* 6  */
-        {OP_Halt,           0, 0, 0},  /* 7  */
+        /* blobSeekToRow() will initialize r[1] to the desired rowid */
+        {OP_NotExists,      0, 5, 1},  /* 2: Seek the cursor to rowid=r[1] */
+        {OP_Column,         0, 0, 1},  /* 3  */
+        {OP_ResultRow,      1, 0, 0},  /* 4  */
+        {OP_Halt,           0, 0, 0},  /* 5  */
      };
      Vdbe *v = (Vdbe *)pBlob->pStmt;
      int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
      VdbeOp *aOp;

-      sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, 
+      sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, 
                           pTab->pSchema->schema_cookie,
                           pTab->pSchema->iGeneration);
      sqlite3VdbeChangeP5(v, 1);     
@@ -285,7 +288,7 @@ int sqlite3_blob_open(
 #else
        aOp[0].p1 = iDb;
        aOp[0].p2 = pTab->tnum;
-        aOp[0].p3 = flags;
+        aOp[0].p3 = wrFlag;
        sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT);
      }
      if( db->mallocFailed==0 ){
@@ -293,7 +296,7 @@ int sqlite3_blob_open(

        /* Remove either the OP_OpenWrite or OpenRead. Set the P2 
        ** parameter of the other to pTab->tnum.  */
-        if( flags ) aOp[1].opcode = OP_OpenWrite;
+        if( wrFlag ) aOp[1].opcode = OP_OpenWrite;
        aOp[1].p2 = pTab->tnum;
        aOp[1].p3 = iDb;   

@@ -306,23 +309,22 @@ int sqlite3_blob_open(
        */
        aOp[1].p4type = P4_INT32;
        aOp[1].p4.i = pTab->nCol+1;
-        aOp[4].p2 = pTab->nCol;
+        aOp[3].p2 = pTab->nCol;

-        pParse->nVar = 1;
+        pParse->nVar = 0;
        pParse->nMem = 1;
        pParse->nTab = 1;
        sqlite3VdbeMakeReady(v, pParse);
      }
    }
   
-    pBlob->flags = flags;
+    pBlob->isPureKV = (pTab->nCol==2 && pTab->iPKey==0);
    pBlob->iCol = iCol;
    pBlob->db = db;
    sqlite3BtreeLeaveAll(db);
    if( db->mallocFailed ){
      goto blob_open_out;
    }
-    sqlite3_bind_int64(pBlob->pStmt, 1, iRow);
    rc = blobSeekToRow(pBlob, iRow, &zErr);
  } while( (++nAttempt)<SQLITE_MAX_SCHEMA_RETRY && rc==SQLITE_SCHEMA );

@@ -61,20 +61,20 @@
 **       ./kvtest run x1 --count 10000 --max-id 1000000
 */
 static const char zHelp[] = 
-"Usage: kvhelp COMMAND ARGS...\n"
+"Usage: kvtest COMMAND ARGS...\n"
 "\n"
-"   kvhelp init DBFILE --count N --size M --pagesize X\n"
+"   kvtest init DBFILE --count N --size M --pagesize X\n"
 "\n"
 "        Generate a new test database file named DBFILE containing N\n"
 "        BLOBs each of size M bytes.  The page size of the new database\n"
 "        file will be X\n"
 "\n"
-"   kvhelp export DBFILE DIRECTORY\n"
+"   kvtest export DBFILE DIRECTORY\n"
 "\n"
 "        Export all the blobs in the kv table of DBFILE into separate\n"
 "        files in DIRECTORY.\n"
 "\n"
-"   kvhelp run DBFILE [options]\n"
+"   kvtest run DBFILE [options]\n"
 "\n"
 "        Run a performance test.  DBFILE can be either the name of a\n"
 "        database or a directory containing sample files.  Options:\n"
@@ -87,6 +87,7 @@ static const char zHelp[] =
 "             --max-id N           Maximum blob key to use\n"
 "             --random             Read blobs in a random order\n"
 "             --start N            Start reading with this blob key\n"
+"             --stats              Output operating stats before exiting\n"
 ;

 /* Reference resources used */
@@ -369,6 +370,122 @@ static sqlite3_int64 timeOfDay(void){
  return t;
 }

+#ifdef __linux__
+/*
+** Attempt to display I/O stats on Linux using /proc/PID/io
+*/
+static void displayLinuxIoStats(FILE *out){
+  FILE *in;
+  char z[200];
+  sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid());
+  in = fopen(z, "rb");
+  if( in==0 ) return;
+  while( fgets(z, sizeof(z), in)!=0 ){
+    static const struct {
+      const char *zPattern;
+      const char *zDesc;
+    } aTrans[] = {
+      { "rchar: ",                  "Bytes received by read():" },
+      { "wchar: ",                  "Bytes sent to write():"    },
+      { "syscr: ",                  "Read() system calls:"      },
+      { "syscw: ",                  "Write() system calls:"     },
+      { "read_bytes: ",             "Bytes read from storage:"  },
+      { "write_bytes: ",            "Bytes written to storage:" },
+      { "cancelled_write_bytes: ",  "Cancelled write bytes:"    },
+    };
+    int i;
+    for(i=0; i<sizeof(aTrans)/sizeof(aTrans[0]); i++){
+      int n = (int)strlen(aTrans[i].zPattern);
+      if( strncmp(aTrans[i].zPattern, z, n)==0 ){
+        fprintf(out, "%-36s %s", aTrans[i].zDesc, &z[n]);
+        break;
+      }
+    }
+  }
+  fclose(in);
+}
+#endif
+
+/*
+** Display memory stats.
+*/
+static int display_stats(
+  sqlite3 *db,                    /* Database to query */
+  int bReset                      /* True to reset SQLite stats */
+){
+  int iCur;
+  int iHiwtr;
+  FILE *out = stdout;
+
+  fprintf(out, "\n");
+
+  iHiwtr = iCur = -1;
+  sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset);
+  fprintf(out,
+          "Memory Used:                         %d (max %d) bytes\n",
+          iCur, iHiwtr);
+  iHiwtr = iCur = -1;
+  sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset);
+  fprintf(out, "Number of Outstanding Allocations:   %d (max %d)\n",
+          iCur, iHiwtr);
+  iHiwtr = iCur = -1;
+  sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset);
+  fprintf(out,
+      "Number of Pcache Pages Used:         %d (max %d) pages\n",
+      iCur, iHiwtr);
+  iHiwtr = iCur = -1;
+  sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset);
+  fprintf(out,
+          "Number of Pcache Overflow Bytes:     %d (max %d) bytes\n",
+          iCur, iHiwtr);
+  iHiwtr = iCur = -1;
+  sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset);
+  fprintf(out,
+      "Number of Scratch Allocations Used:  %d (max %d)\n",
+      iCur, iHiwtr);
+  iHiwtr = iCur = -1;
+  sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset);
+  fprintf(out,
+          "Number of Scratch Overflow Bytes:    %d (max %d) bytes\n",
+          iCur, iHiwtr);
+  iHiwtr = iCur = -1;
+  sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset);
+  fprintf(out, "Largest Allocation:                  %d bytes\n",
+          iHiwtr);
+  iHiwtr = iCur = -1;
+  sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset);
+  fprintf(out, "Largest Pcache Allocation:           %d bytes\n",
+          iHiwtr);
+  iHiwtr = iCur = -1;
+  sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset);
+  fprintf(out, "Largest Scratch Allocation:          %d bytes\n",
+          iHiwtr);
+
+  iHiwtr = iCur = -1;
+  sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset);
+  fprintf(out, "Pager Heap Usage:                    %d bytes\n",
+      iCur);
+  iHiwtr = iCur = -1;
+  sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1);
+  fprintf(out, "Page cache hits:                     %d\n", iCur);
+  iHiwtr = iCur = -1;
+  sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1);
+  fprintf(out, "Page cache misses:                   %d\n", iCur);
+  iHiwtr = iCur = -1;
+  sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1);
+  fprintf(out, "Page cache writes:                   %d\n", iCur);
+  iHiwtr = iCur = -1;
+
+#ifdef __linux__
+  displayLinuxIoStats(out);
+#endif
+
+  /* Do not remove this machine readable comment: extra-stats-output-here */
+
+  fprintf(out, "\n");
+  return 0;
+}
+
 /* Blob access order */
 #define ORDER_ASC     1
 #define ORDER_DESC    2
@@ -389,6 +506,7 @@ static int runMain(int argc, char **argv){
  int iPagesize = 0;          /* Database page size */
  int iCache = 1000;          /* Database cache size in kibibytes */
  int bBlobApi = 0;           /* Use the incremental blob I/O API */
+  int bStats = 0;             /* Print stats before exiting */
  int eOrder = ORDER_ASC;     /* Access order */
  sqlite3 *db = 0;            /* Database connection */
  sqlite3_stmt *pStmt = 0;    /* Prepared statement for SQL access */
@@ -398,6 +516,7 @@ static int runMain(int argc, char **argv){
  int nData = 0;              /* Bytes of data */
  sqlite3_int64 nTotal = 0;   /* Total data read */
  unsigned char *pData;       /* Content of the blob */
+  int nAlloc = 0;             /* Space allocated for pData[] */
  

  assert( strcmp(argv[1],"run")==0 );
@@ -449,6 +568,10 @@ static int runMain(int argc, char **argv){
      bBlobApi = 1;
      continue;
    }
+    if( strcmp(z, "-stats")==0 ){
+      bStats = 1;
+      continue;
+    }
    fatalError("unknown option: \"%s\"", argv[i]);
  }
  tmStart = timeOfDay();
@@ -499,14 +622,16 @@ static int runMain(int argc, char **argv){
      }
      if( rc==SQLITE_OK ){
        nData = sqlite3_blob_bytes(pBlob);
-        pData = sqlite3_malloc( nData+1 );
+        if( nAlloc<nData+1 ){
+          nAlloc = nData+100;
+          pData = sqlite3_realloc(pData, nAlloc);
+        }
        if( pData==0 ) fatalError("cannot allocate %d bytes", nData+1);
        rc = sqlite3_blob_read(pBlob, pData, nData, 0);
        if( rc!=SQLITE_OK ){
          fatalError("could not read the blob at %d: %s", iKey,
                     sqlite3_errmsg(db));
        }
-        sqlite3_free(pData);
      }
    }else{
      /* CASE 3: Reading from database using SQL */
@@ -540,8 +665,12 @@ static int runMain(int argc, char **argv){
    nTotal += nData;
    if( nData==0 ){ nCount++; nExtra++; }
  }
+  if( nAlloc ) sqlite3_free(pData);
  if( pStmt ) sqlite3_finalize(pStmt);
  if( pBlob ) sqlite3_blob_close(pBlob);
+  if( bStats ){
+    display_stats(db, 0);
+  }
  if( db ) sqlite3_close(db);
  tmElapsed = timeOfDay() - tmStart;
  if( nExtra ){
@@ -61,9 +61,6 @@ static const char zHelp[] =
 #if SQLITE_VERSION_NUMBER<3005000
 # define sqlite3_int64 sqlite_int64
 #endif
-#ifdef SQLITE_ENABLE_RBU
-# include "sqlite3rbu.h"
-#endif

 /* All global state is held in this structure */
 static struct Global {
@@ -1181,10 +1178,10 @@ void testset_rtree(int p1, int p2){
  unsigned mxCoord;
  unsigned x0, x1, y0, y1, z0, z1;
  unsigned iStep;
-  int *aCheck = sqlite3_malloc( sizeof(int)*g.szTest*100 );
+  int *aCheck = sqlite3_malloc( sizeof(int)*g.szTest*500 );

  mxCoord = 15000;
-  n = g.szTest*100;
+  n = g.szTest*500;
  speedtest1_begin_test(100, "%d INSERTs into an r-tree", n);
  speedtest1_exec("BEGIN");
  speedtest1_exec("CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1,z0,z1)");
@@ -1207,11 +1204,11 @@ void testset_rtree(int p1, int p2){
  speedtest1_end_test();

  speedtest1_begin_test(101, "Copy from rtree to a regular table");
-  speedtest1_exec(" TABLE t1(id INTEGER PRIMARY KEY,x0,x1,y0,y1,z0,z1)");
+  speedtest1_exec("CREATE TABLE t1(id INTEGER PRIMARY KEY,x0,x1,y0,y1,z0,z1)");
  speedtest1_exec("INSERT INTO t1 SELECT * FROM rt1");
  speedtest1_end_test();

-  n = g.szTest*20;
+  n = g.szTest*100;
  speedtest1_begin_test(110, "%d one-dimensional intersect slice queries", n);
  speedtest1_prepare("SELECT count(*) FROM rt1 WHERE x0>=?1 AND x1<=?2");
  iStep = mxCoord/n;
@@ -1224,7 +1221,7 @@ void testset_rtree(int p1, int p2){
  speedtest1_end_test();

  if( g.bVerify ){
-    n = g.szTest*20;
+    n = g.szTest*100;
    speedtest1_begin_test(111, "Verify result from 1-D intersect slice queries");
    speedtest1_prepare("SELECT count(*) FROM t1 WHERE x0>=?1 AND x1<=?2");
    iStep = mxCoord/n;
@@ -1240,7 +1237,7 @@ void testset_rtree(int p1, int p2){
    speedtest1_end_test();
  }
  
-  n = g.szTest*20;
+  n = g.szTest*100;
  speedtest1_begin_test(120, "%d one-dimensional overlap slice queries", n);
  speedtest1_prepare("SELECT count(*) FROM rt1 WHERE y1>=?1 AND y0<=?2");
  iStep = mxCoord/n;
@@ -1253,7 +1250,7 @@ void testset_rtree(int p1, int p2){
  speedtest1_end_test();

  if( g.bVerify ){
-    n = g.szTest*20;
+    n = g.szTest*100;
    speedtest1_begin_test(121, "Verify result from 1-D overlap slice queries");
    speedtest1_prepare("SELECT count(*) FROM t1 WHERE y1>=?1 AND y0<=?2");
    iStep = mxCoord/n;
@@ -1270,7 +1267,7 @@ void testset_rtree(int p1, int p2){
  }
  

-  n = g.szTest*20;
+  n = g.szTest*100;
  speedtest1_begin_test(125, "%d custom geometry callback queries", n);
  sqlite3_rtree_geometry_callback(g.db, "xslice", xsliceGeometryCallback, 0);
  speedtest1_prepare("SELECT count(*) FROM rt1 WHERE id MATCH xslice(?1,?2)");
@@ -1286,7 +1283,7 @@ void testset_rtree(int p1, int p2){
  }
  speedtest1_end_test();

-  n = g.szTest*80;
+  n = g.szTest*400;
  speedtest1_begin_test(130, "%d three-dimensional intersect box queries", n);
  speedtest1_prepare("SELECT count(*) FROM rt1 WHERE x1>=?1 AND x0<=?2"
                     " AND y1>=?1 AND y0<=?2 AND z1>=?1 AND z0<=?2");
@@ -1299,7 +1296,7 @@ void testset_rtree(int p1, int p2){
  }
  speedtest1_end_test();

-  n = g.szTest*100;
+  n = g.szTest*500;
  speedtest1_begin_test(140, "%d rowid queries", n);
  speedtest1_prepare("SELECT * FROM rt1 WHERE id=?1");
  for(i=1; i<=n; i++){
@@ -1458,11 +1455,6 @@ int main(int argc, char **argv){
        noSync = 1;
      }else if( strcmp(z,"notnull")==0 ){
        g.zNN = "NOT NULL";
-#ifdef SQLITE_ENABLE_RBU
-      }else if( strcmp(z,"rbu")==0 ){
-        sqlite3ota_create_vfs("rbu", 0);
-        sqlite3_vfs_register(sqlite3_vfs_find("rbu"), 1);
-#endif
      }else if( strcmp(z,"pagesize")==0 ){
        if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);
        pageSize = integerValue(argv[++i]);
@@ -121,29 +121,43 @@ if {$tcl_platform(platform)!="windows" || \
    SELECT name FROM fsdir WHERE dir = '.' AND name = '.'
  } {test.db .}

+  proc sort_files { names {nocase false} } {
+    if {$nocase && $::tcl_platform(platform) eq "windows"} {
+      return [lsort -nocase $names]
+    } else {
+      return [lsort $names]
+    }
+  }
+
  proc list_root_files {} {
    if {$::tcl_platform(platform) eq "windows"} {
-      set res [list]
-      foreach name [glob -directory $::env(fstreeDrive)/ -- *] {
+      set res [list]; set dir $::env(fstreeDrive)/; set names [list]
+      eval lappend names [glob -nocomplain -directory $dir -- *]
+      foreach name $names {
        if {[string index [file tail $name] 0] eq "."} continue
+        if {[file attributes $name -hidden]} continue
+        if {[file attributes $name -system]} continue
        lappend res $name
      }
-      return $res
+      return [sort_files $res true]
    } else {
-      return [string map {/ {}} [glob /*]]
+      return [sort_files [string map {/ {}} [glob -nocomplain -- /*]]]
    }
  }

  proc list_files { pattern } {
    if {$::tcl_platform(platform) eq "windows"} {
-      set res [list]
-      foreach name [glob -nocomplain $pattern] {
+      set res [list]; set names [list]
+      eval lappend names [glob -nocomplain -- $pattern]
+      foreach name $names {
        if {[string index [file tail $name] 0] eq "."} continue
+        if {[file attributes $name -hidden]} continue
+        if {[file attributes $name -system]} continue
        lappend res $name
      }
-      return $res
+      return [sort_files $res]
    } else {
-      return [glob -nocomplain $pattern]
+      return [sort_files [glob -nocomplain -- $pattern]]
    }
  }

@@ -153,18 +167,19 @@ if {$tcl_platform(platform)!="windows" || \
  #
  set res [list]
  set root_files [list_root_files]
-  set num_root_files [llength $root_files]
-  set lim_root_files [expr {$num_root_files > 5 ? 5 : $num_root_files}]
-  foreach p [lrange $root_files 0 [expr {$lim_root_files - 1}]] {
+  foreach p $root_files {
    if {$::tcl_platform(platform) eq "windows"} {
-      if {[regexp {\$} $p]} {incr lim_root_files -1} else {lappend res $p}
+      if {![regexp {\$} $p]} {lappend res $p}
    } else {
      lappend res "/$p"
    }
  }
-  do_execsql_test 3.1 [subst {
-    SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $lim_root_files;
-  }] $res
+  set num_root_files [llength $root_files]
+  do_test 3.1 {
+    sort_files [execsql {
+      SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $num_root_files;
+    }] true
+  } [sort_files $res true]

  # Read all entries in the current directory.
  #
@@ -182,7 +197,7 @@ if {$tcl_platform(platform)!="windows" || \
  set res [contents $pwd]
  do_execsql_test 3.2 {
    SELECT path FROM fstree WHERE path GLOB $pwd ORDER BY 1
-  } [lsort $res]
+  } [sort_files $res]

  # Add some sub-directories and files to the current directory.
  #
@@ -23,6 +23,7 @@ NAME=$1
 shift
 #CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
 CC_OPTS="-DSQLITE_ENABLE_MEMSYS5"
+CC=gcc
 SPEEDTEST_OPTS="--shrink-memory --reprepare --stats --heap 10000000 64"
 SIZE=5
 LEAN_OPTS="-DSQLITE_THREADSAFE=0"
@@ -80,11 +81,19 @@ while test "$1" != ""; do
    --lean)
        CC_OPTS="$CC_OPTS $LEAN_OPTS"
        ;;
+    --clang)
+        CC=clang
+        ;;
    --heap)
        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5"
        shift;
        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64"
        ;;
+    --lookaside)
+        shift;
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --lookaside $1 $2"
+        shift;
+        ;;
    --repeat)
        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RCACHE"
        shift;
@@ -94,6 +103,10 @@ while test "$1" != ""; do
        shift;
        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --mmap $1"
        ;;
+    --rtree)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset rtree"
+        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RTREE"
+        ;;
    *)
        CC_OPTS="$CC_OPTS $1"
        ;;
@@ -105,15 +118,15 @@ echo "NAME           = $NAME" | tee summary-$NAME.txt
 echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
 echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
 rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
-gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
+$CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
 size sqlite3.o | tee -a summary-$NAME.txt
 if test $doExplain -eq 1; then
-  gcc -g -Os -Wall -I. $CC_OPTS \
+  $CC -g -Os -Wall -I. $CC_OPTS \
     -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
    ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
 fi
 SRC=./speedtest1.c
-gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
+$CC -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
 ls -l speedtest1 | tee -a summary-$NAME.txt
 if test $doCachegrind -eq 1; then
  valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
Author	SHA1	Message	Date
dan	e8814cc83c	Fix problems in the previous commit. FossilOrigin-Name: 74ad80eb745c708bdbbbf24cbbbf50971e84182f	2017-01-21 16:21:24 +00:00
dan	0fbee29700	Add extra (somewhat inefficient) trace callbacks for triggers if SQLITE_TRACE_TRIGGER is defined. FossilOrigin-Name: ffda1d1e1c858abd02f0c07b906cfac5ad075498	2017-01-21 15:58:42 +00:00
drh	184d902db1	In the kvtest.c test utility, reuse the buffer into which blobs are read, rather than reallocating it for each row. This is a closer match to how other test programs work, and thus provides a better comparison. FossilOrigin-Name: 0d1ad13a296b22d6fe36879b56f99bd6af1acd3a	2017-01-21 15:55:41 +00:00
drh	36cae856ee	Remove an unnecessary sqlite3_bind_int64() call from sqlite3_blob_open(). Also other minor refactoring of the sqlite3_blob implementation. FossilOrigin-Name: 9d197a532349f4b1caf66bbed70ca46df86cb86f	2017-01-21 14:11:28 +00:00
drh	cd64553015	Minor performance optimization and size reduction to the accessPayload() routine in btree.c. FossilOrigin-Name: 264e5c10d7144910b3223b64546567fa20e4bc65	2017-01-20 20:43:14 +00:00
dan	dfd0de82f1	Fix a typo in the help message for kvtest. FossilOrigin-Name: 8971d98f25a4f5fb060db8ed6a4b06f083122a50	2017-01-20 16:47:34 +00:00
dan	befcd8ad84	Add option "--stats" to test program kvtest. Specifying --stats causes kvtest to output information similar to the shell tool option of the same name. FossilOrigin-Name: 90291327fc127671d9847a4a2ce1ed47a408cfc6	2017-01-20 16:46:20 +00:00
drh	508286701a	Get the "--testset rtree" option working on speedtest1. Add the --rtree, --lookaside, and --clang options to the speed-check.sh script. FossilOrigin-Name: 87b640c8d07a76b2bc7e896e01965cc09e06f77b	2017-01-20 16:09:12 +00:00
drh	4df65fc20f	Minor performance optimizations to sqlite3_blob_open() and sqlite3_blob_reopen(). FossilOrigin-Name: 52a61967d920047ea0b4409b79793e05c0128964	2017-01-20 00:40:26 +00:00
drh	8674e49214	If compiled with SQLITE_INLINE_MEMCPY, all memcpy() calls are replaced with in-line code. With that change, cachegrind shows which memcpy() calls are taking the most time. This is a performance-measurement hack only and is not for production use. FossilOrigin-Name: 9ed38521617136223a667988aed40e25797faf84	2017-01-19 21:20:11 +00:00
mistachkin	2f31d02049	In the 'vtshim' extension, avoid accessing freed memory when handling errors from xCreate/xConnect. FossilOrigin-Name: ffd559afd32dcdce9c733ebccdee88fda9b689cf	2017-01-19 18:20:36 +00:00
mistachkin	aac853bae8	In the 'windirent' test module, use a macro for the hidden/system attribute checking. FossilOrigin-Name: a84a08d0716656dc0b26eafb1841c48d83c67ef2	2017-01-18 22:47:42 +00:00
mistachkin	d3e73d6c1c	Remove superfluous option to Tcl 'lsort' in the vtabH test file. FossilOrigin-Name: b92cc6e58ae31cbe6600a522beb5485f7add04b2	2017-01-18 22:19:01 +00:00
mistachkin	7059138a08	Make the vtabH-3.1 test more portable and robust. FossilOrigin-Name: d3c91c1fb345fbcbfc60a897bebf771c795430c9	2017-01-18 22:16:34 +00:00
mistachkin	8a9e83583b	Fix handling of initial hidden and/or system files in the opendir() implementation for Windows. No changes to non-test code. FossilOrigin-Name: 26dd42b462dc621b8b0a2295fc91d3e61ac732b6	2017-01-18 22:16:20 +00:00
dan	25fd2e247b	Fix a problem that could cause a spurious SQLITE_NOMEM error when attempting to resume an RBU operation if the previous client failed right after completing the incremental checkpoint. Also a "cannot vacuum wal db" error that could occur when resuming an RBU vacuum if an error (OOM or IO error) occurs during the incremental checkpoint. FossilOrigin-Name: 681d96eb822e606da53700867191d4738bda20c8	2017-01-17 10:41:42 +00:00
drh	c711f53f30	Disable intrinsic functions for Windows using Clang, due to reports of linkage errors. This causes a 0.6% performance reduction. We will want to revisit this change in the future. FossilOrigin-Name: 7fd560c6d2ff470b755ad118287a0a8825b3009e	2017-01-17 00:10:58 +00:00
drh	de48a3f98a	Back out check-in [0b3174e0b1364c] and replace it with a better fix for \ticket [91e2e8ba6ff2e2] - a fix that does not cause the problem identified by ticket [7ffd1ca1d2ad4ec]. Add new test cases for both tickets. FossilOrigin-Name: 9b64af7b5201a8700ae9e384b04714ca18df7449	2017-01-16 18:10:17 +00:00