Add the sqlite3rbu_state() API. Used to determine the current state (creating OAL, ready to move OAL, incremental-checkpoint, finished or error) of an RBU operation.

FossilOrigin-Name: 92e7df0ff5c4c118c63d92a767dc82700438a310

Makefile.in

@@ -417,6 +417,7 @@ TESTSRC = \
 TESTSRC += \
   $(TOP)/ext/misc/amatch.c \
   $(TOP)/ext/misc/closure.c \
+  $(TOP)/ext/misc/csv.c \
   $(TOP)/ext/misc/eval.c \
   $(TOP)/ext/misc/fileio.c \
   $(TOP)/ext/misc/fuzzer.c \
@@ -538,7 +539,8 @@ TESTPROGS = \
   testfixture$(TEXE) \
   sqlite3$(TEXE) \
   sqlite3_analyzer$(TEXE) \
-  sqldiff$(TEXE)
+  sqldiff$(TEXE) \
+  dbhash$(TEXE)
 
 # Databases containing fuzzer test cases
 #
@@ -586,8 +588,15 @@ sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
 		$(TOP)/src/shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
 
-sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
-	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
+sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
+	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)
+
+dbhash$(TEXE):	$(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h
+	$(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS)
+
+scrub$(TEXE):	$(TOP)/ext/misc/scrub.c sqlite3.lo
+	$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
+		$(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS)
 
 srcck1$(BEXE):	$(TOP)/tool/srcck1.c
 	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
@@ -602,8 +611,8 @@ fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
 
-mptester$(TEXE):	sqlite3.c $(TOP)/mptest/mptest.c
-	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
+mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
+	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
 		$(TLIBS) -rpath "$(libdir)"
 
 MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
@@ -1147,8 +1156,8 @@ rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
 LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
 
-wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.c
-	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS)
+wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)
 
 speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
@@ -1165,7 +1174,7 @@ loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
 #
 VALIDIDS=' sqlite3(changeset|changegroup|session)?_'
 checksymbols: sqlite3.lo
-	nm -g --defined-only sqlite3.o | egrep -v $(VALIDIDS); test $$? -ne 0
+	nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0
 	echo '0 errors out of 1 tests'
 
 # Build the amalgamation-autoconf package.  The amalamgation-tarball target builds
@@ -1246,6 +1255,7 @@ clean:
 	rm -f fuzzershell fuzzershell.exe
 	rm -f fuzzcheck fuzzcheck.exe
 	rm -f sqldiff sqldiff.exe
+	rm -f dbhash dbhash.exe
 	rm -f fts5.* fts5parse.*
 
 distclean:	clean

Makefile.msc

@@ -1293,6 +1293,7 @@ TESTSRC = \
 TESTEXT = \
   $(TOP)\ext\misc\amatch.c \
   $(TOP)\ext\misc\closure.c \
+  $(TOP)\ext\misc\csv.c \
   $(TOP)\ext\misc\eval.c \
   $(TOP)\ext\misc\fileio.c \
   $(TOP)\ext\misc\fuzzer.c \
@@ -1377,7 +1378,8 @@ TESTPROGS = \
   testfixture.exe \
   $(SQLITE3EXE) \
   sqlite3_analyzer.exe \
-  sqldiff.exe
+  sqldiff.exe \
+  dbhash.exe
 
 # Databases containing fuzzer test cases
 #
@@ -1455,6 +1457,12 @@ $(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_S
 sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+dbhash.exe:	$(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
+scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 srcck1.exe:	$(TOP)\tool\srcck1.c
 	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
 
@@ -2101,6 +2109,6 @@ clean:
 	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
-	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
+	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
 	del /Q fts5.* fts5parse.* 2>NUL
 # <</mark>>

VERSION

@@ -1 +1 @@
-3.13.0
+3.14.0

autoconf/configure.ac

@@ -30,39 +30,52 @@ AC_FUNC_STRERROR_R
 AC_CONFIG_FILES([Makefile sqlite3.pc])
 AC_SUBST(BUILD_CFLAGS)
 
-#-----------------------------------------------------------------------
+#-------------------------------------------------------------------------
+# Two options to enable readline compatible libraries: 
+#
 #   --enable-editline
 #   --enable-readline
 #
-AC_ARG_ENABLE(editline, [AS_HELP_STRING(
-  [--enable-editline], 
-  [use BSD libedit])], 
-  [], [enable_editline=yes])
-AC_ARG_ENABLE(readline, [AS_HELP_STRING(
-  [--enable-readline], 
-  [use readline])], 
-  [], [enable_readline=no])
-if test x"$enable_editline" != xno ; then
-  sLIBS=$LIBS
-  LIBS=""
-  AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no])
-  READLINE_LIBS=$LIBS
-  if test x"$LIBS" != "x"; then
-     AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
-  else
-    unset ac_cv_search_readline
-  fi
-  LIBS=$sLIBS
-fi
-if test x"$enable_readline" != xno ; then
-  sLIBS=$LIBS
-  LIBS=""
-  AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], [])
-  AC_SEARCH_LIBS(readline, readline, [], [enable_readline=no])
-  AC_CHECK_FUNCS(readline, [], [])
-  READLINE_LIBS=$LIBS
-  LIBS=$sLIBS
-fi
+# Both are enabled by default. If, after command line processing both are
+# still enabled, the script searches for editline first and automatically
+# disables readline if it is found. So, to use readline explicitly, the
+# user must pass "--disable-editline". To disable command line editing
+# support altogether, "--disable-editline --disable-readline".
+#
+# When searching for either library, check for headers before libraries 
+# as some distros supply packages that contain libraries but not header
+# files, which come as a separate development package.
+#
+AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])])
+AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])])
+
+AS_IF([ test x"$enable_editline" != xno ],[
+  AC_CHECK_HEADERS([editline/readline.h],[
+    sLIBS=$LIBS
+    LIBS=""
+    AC_SEARCH_LIBS([readline],[edit],[
+      AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
+      READLINE_LIBS=$LIBS
+      enable_readline=no
+    ])
+    AS_UNSET(ac_cv_search_readline)
+    LIBS=$sLIBS
+  ])
+])
+
+AS_IF([ test x"$enable_readline" != xno ],[
+  AC_CHECK_HEADERS([readline/readline.h],[
+    sLIBS=$LIBS
+    LIBS=""
+    AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], [])
+    AC_SEARCH_LIBS(readline,[readline edit], [
+      AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper)
+      READLINE_LIBS=$LIBS
+    ])
+    LIBS=$sLIBS
+  ])
+])
+
 AC_SUBST(READLINE_LIBS)
 #-----------------------------------------------------------------------
 

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.13.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.14.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.13.0'
-PACKAGE_STRING='sqlite 3.13.0'
+PACKAGE_VERSION='3.14.0'
+PACKAGE_STRING='sqlite 3.14.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1461,7 +1461,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.13.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.14.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1526,7 +1526,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.13.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.14.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1648,7 +1648,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.13.0
+sqlite configure 3.14.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2067,7 +2067,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.13.0, which was
+It was created by sqlite $as_me 3.14.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -10757,6 +10757,20 @@ else
       fi
     fi
 
+    # Recent versions of Xcode on Macs hid the tclConfig.sh file
+    # in a strange place.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
     # then check for a private Tcl installation
     if test x"${ac_cv_c_tclconfig}" = x ; then
       for i in \
@@ -12095,7 +12109,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.13.0, which was
+This file was extended by sqlite $as_me 3.14.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12161,7 +12175,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.13.0
+sqlite config.status 3.14.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -334,6 +334,20 @@ if test "${use_tcl}" = "yes" ; then
       fi
     fi
 
+    # Recent versions of Xcode on Macs hid the tclConfig.sh file
+    # in a strange place.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
     # then check for a private Tcl installation
     if test x"${ac_cv_c_tclconfig}" = x ; then
       for i in \

ext/fts5/fts5Int.h

@@ -686,7 +686,6 @@ int sqlite3Fts5ExprPopulatePoslists(
     Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 );
 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
-void sqlite3Fts5ExprClearEof(Fts5Expr*);
 
 int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 

ext/fts5/fts5_expr.c

@@ -2617,17 +2617,6 @@ void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
   fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
 }
 
-static void fts5ExprClearEof(Fts5ExprNode *pNode){
-  int i;
-  for(i=0; i<pNode->nChild; i++){
-    fts5ExprClearEof(pNode->apChild[i]);
-  }
-  pNode->bEof = 0;
-}
-void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
-  fts5ExprClearEof(pExpr->pRoot);
-}
-
 /*
 ** This function is only called for detail=columns tables. 
 */

ext/fts5/fts5_main.c

@@ -1186,7 +1186,6 @@ static int fts5FilterMethod(
     pCsr->ePlan = FTS5_PLAN_SOURCE;
     pCsr->pExpr = pTab->pSortCsr->pExpr;
     rc = fts5CursorFirst(pTab, pCsr, bDesc);
-    sqlite3Fts5ExprClearEof(pCsr->pExpr);
   }else if( pMatch ){
     const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
     if( zExpr==0 ) zExpr = "";

ext/fts5/test/fts5rank.test

@@ -91,9 +91,38 @@ do_test 2.7 {
 } {1 3 2}
 
 
+#--------------------------------------------------------------------------
+# At one point there was a problem with queries such as:
+#
+#   ... MATCH 'x OR y' ORDER BY rank;
+#
+# if there were zero occurrences of token 'y' in the dataset. The
+# following tests verify that that problem has been addressed.
+#
+foreach_detail_mode $::testprefix {
+  do_execsql_test 3.0 {
+    CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%);
+    INSERT INTO y1 VALUES('test xyz');
+    INSERT INTO y1 VALUES('test test xyz test');
+    INSERT INTO y1 VALUES('test test xyz');
+  }
 
+  do_execsql_test 3.1 {
+    SELECT rowid FROM y1('test OR tset');
+  } {1 2 3}
 
+  do_execsql_test 3.2 {
+    SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1)
+  } {2 3 1}
 
+  do_execsql_test 3.3 {
+    SELECT rowid FROM y1('test OR tset') ORDER BY +rank
+  } {2 3 1}
+
+  do_execsql_test 3.4 {
+    SELECT rowid FROM y1('test OR tset') ORDER BY rank
+  } {2 3 1}
+}
 
 
 finish_test

ext/misc/csv.c

@@ -0,0 +1,861 @@
+/*
+** 2016-05-28
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains the implementation of an SQLite virtual table for
+** reading CSV files.
+**
+** Usage:
+**
+**    .load ./csv
+**    CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
+**    SELECT * FROM csv;
+**
+** The columns are named "c1", "c2", "c3", ... by default.  But the
+** application can define its own CREATE TABLE statement as an additional
+** parameter.  For example:
+**
+**    CREATE VIRTUAL TABLE temp.csv2 USING csv(
+**       filename = "../http.log",
+**       schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)"
+**    );
+**
+** Instead of specifying a file, the text of the CSV can be loaded using
+** the data= parameter.
+**
+** If the columns=N parameter is supplied, then the CSV file is assumed to have
+** N columns.  If the columns parameter is omitted, the CSV file is opened
+** as soon as the virtual table is constructed and the first row of the CSV
+** is read in order to count the tables.
+**
+** Some extra debugging features (used for testing virtual tables) are available
+** if this module is compiled with -DSQLITE_TEST.
+*/
+#include <sqlite3ext.h>
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <stdio.h>
+
+/*
+** A macro to hint to the compiler that a function should not be
+** inlined.
+*/
+#if defined(__GNUC__)
+#  define CSV_NOINLINE  __attribute__((noinline))
+#elif defined(_MSC_VER) && _MSC_VER>=1310
+#  define CSV_NOINLINE  __declspec(noinline)
+#else
+#  define CSV_NOINLINE
+#endif
+
+
+/* Max size of the error message in a CsvReader */
+#define CSV_MXERR 200
+
+/* Size of the CsvReader input buffer */
+#define CSV_INBUFSZ 1024
+
+/* A context object used when read a CSV file. */
+typedef struct CsvReader CsvReader;
+struct CsvReader {
+  FILE *in;              /* Read the CSV text from this input stream */
+  char *z;               /* Accumulated text for a field */
+  int n;                 /* Number of bytes in z */
+  int nAlloc;            /* Space allocated for z[] */
+  int nLine;             /* Current line number */
+  char cTerm;            /* Character that terminated the most recent field */
+  size_t iIn;            /* Next unread character in the input buffer */
+  size_t nIn;            /* Number of characters in the input buffer */
+  char *zIn;             /* The input buffer */
+  char zErr[CSV_MXERR];  /* Error message */
+};
+
+/* Initialize a CsvReader object */
+static void csv_reader_init(CsvReader *p){
+  p->in = 0;
+  p->z = 0;
+  p->n = 0;
+  p->nAlloc = 0;
+  p->nLine = 0;
+  p->nIn = 0;
+  p->zIn = 0;
+  p->zErr[0] = 0;
+}
+
+/* Close and reset a CsvReader object */
+static void csv_reader_reset(CsvReader *p){
+  if( p->in ){
+    fclose(p->in);
+    sqlite3_free(p->zIn);
+  }
+  sqlite3_free(p->z);
+  csv_reader_init(p);
+}
+
+/* Report an error on a CsvReader */
+static void csv_errmsg(CsvReader *p, const char *zFormat, ...){
+  va_list ap;
+  va_start(ap, zFormat);
+  sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap);
+  va_end(ap);
+}
+
+/* Open the file associated with a CsvReader
+** Return the number of errors.
+*/
+static int csv_reader_open(
+  CsvReader *p,               /* The reader to open */
+  const char *zFilename,      /* Read from this filename */
+  const char *zData           /*  ... or use this data */
+){
+  if( zFilename ){
+    p->zIn = sqlite3_malloc( CSV_INBUFSZ );
+    if( p->zIn==0 ){
+      csv_errmsg(p, "out of memory");
+      return 1;
+    }
+    p->in = fopen(zFilename, "rb");
+    if( p->in==0 ){
+      csv_reader_reset(p);
+      csv_errmsg(p, "cannot open '%s' for reading", zFilename);
+      return 1;
+    }
+  }else{
+    assert( p->in==0 );
+    p->zIn = (char*)zData;
+    p->nIn = strlen(zData);
+  }
+  return 0;
+}
+
+/* The input buffer has overflowed.  Refill the input buffer, then
+** return the next character
+*/
+static CSV_NOINLINE int csv_getc_refill(CsvReader *p){
+  size_t got;
+
+  assert( p->iIn>=p->nIn );  /* Only called on an empty input buffer */
+  assert( p->in!=0 );        /* Only called if reading froma file */
+
+  got = fread(p->zIn, 1, CSV_INBUFSZ, p->in);
+  if( got==0 ) return EOF;
+  p->nIn = got;
+  p->iIn = 1;
+  return p->zIn[0];
+}
+
+/* Return the next character of input.  Return EOF at end of input. */
+static int csv_getc(CsvReader *p){
+  if( p->iIn >= p->nIn ){
+    if( p->in!=0 ) return csv_getc_refill(p);
+    return EOF;
+  }
+  return p->zIn[p->iIn++];
+}
+
+/* Increase the size of p->z and append character c to the end. 
+** Return 0 on success and non-zero if there is an OOM error */
+static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){
+  char *zNew;
+  int nNew = p->nAlloc*2 + 100;
+  zNew = sqlite3_realloc64(p->z, nNew);
+  if( zNew ){
+    p->z = zNew;
+    p->nAlloc = nNew;
+    p->z[p->n++] = c;
+    return 0;
+  }else{
+    csv_errmsg(p, "out of memory");
+    return 1;
+  }
+}
+
+/* Append a single character to the CsvReader.z[] array.
+** Return 0 on success and non-zero if there is an OOM error */
+static int csv_append(CsvReader *p, char c){
+  if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c);
+  p->z[p->n++] = c;
+  return 0;
+}
+
+/* Read a single field of CSV text.  Compatible with rfc4180 and extended
+** with the option of having a separator other than ",".
+**
+**   +  Input comes from p->in.
+**   +  Store results in p->z of length p->n.  Space to hold p->z comes
+**      from sqlite3_malloc64().
+**   +  Keep track of the line number in p->nLine.
+**   +  Store the character that terminates the field in p->cTerm.  Store
+**      EOF on end-of-file.
+**
+** Return "" at EOF.  Return 0 on an OOM error.
+*/
+static char *csv_read_one_field(CsvReader *p){
+  int c;
+  p->n = 0;
+  c = csv_getc(p);
+  if( c==EOF ){
+    p->cTerm = EOF;
+    return "";
+  }
+  if( c=='"' ){
+    int pc, ppc;
+    int startLine = p->nLine;
+    pc = ppc = 0;
+    while( 1 ){
+      c = csv_getc(p);
+      if( c<='"' || pc=='"' ){
+        if( c=='\n' ) p->nLine++;
+        if( c=='"' ){
+          if( pc=='"' ){
+            pc = 0;
+            continue;
+          }
+        }
+        if( (c==',' && pc=='"')
+         || (c=='\n' && pc=='"')
+         || (c=='\n' && pc=='\r' && ppc=='"')
+         || (c==EOF && pc=='"')
+        ){
+          do{ p->n--; }while( p->z[p->n]!='"' );
+          p->cTerm = c;
+          break;
+        }
+        if( pc=='"' && c!='\r' ){
+          csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"');
+          break;
+        }
+        if( c==EOF ){
+          csv_errmsg(p, "line %d: unterminated %c-quoted field\n",
+                     startLine, '"');
+          p->cTerm = c;
+          break;
+        }
+      }
+      if( csv_append(p, (char)c) ) return 0;
+      ppc = pc;
+      pc = c;
+    }
+  }else{
+    while( c>',' || (c!=EOF && c!=',' && c!='\n') ){
+      if( csv_append(p, (char)c) ) return 0;
+      c = csv_getc(p);
+    }
+    if( c=='\n' ){
+      p->nLine++;
+      if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--;
+    }
+    p->cTerm = c;
+  }
+  if( p->z ) p->z[p->n] = 0;
+  return p->z;
+}
+
+
+/* Forward references to the various virtual table methods implemented
+** in this file. */
+static int csvtabCreate(sqlite3*, void*, int, const char*const*, 
+                           sqlite3_vtab**,char**);
+static int csvtabConnect(sqlite3*, void*, int, const char*const*, 
+                           sqlite3_vtab**,char**);
+static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*);
+static int csvtabDisconnect(sqlite3_vtab*);
+static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**);
+static int csvtabClose(sqlite3_vtab_cursor*);
+static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr,
+                          int argc, sqlite3_value **argv);
+static int csvtabNext(sqlite3_vtab_cursor*);
+static int csvtabEof(sqlite3_vtab_cursor*);
+static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int);
+static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*);
+
+/* An instance of the CSV virtual table */
+typedef struct CsvTable {
+  sqlite3_vtab base;              /* Base class.  Must be first */
+  char *zFilename;                /* Name of the CSV file */
+  char *zData;                    /* Raw CSV data in lieu of zFilename */
+  long iStart;                    /* Offset to start of data in zFilename */
+  int nCol;                       /* Number of columns in the CSV file */
+  unsigned int tstFlags;          /* Bit values used for testing */
+} CsvTable;
+
+/* Allowed values for tstFlags */
+#define CSVTEST_FIDX  0x0001      /* Pretend that constrained searchs cost less*/
+
+/* A cursor for the CSV virtual table */
+typedef struct CsvCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  CsvReader rdr;                  /* The CsvReader object */
+  char **azVal;                   /* Value of the current row */
+  int *aLen;                      /* Length of each entry */
+  sqlite3_int64 iRowid;           /* The current rowid.  Negative for EOF */
+} CsvCursor;
+
+/* Transfer error message text from a reader into a CsvTable */
+static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){
+  sqlite3_free(pTab->base.zErrMsg);
+  pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr);
+}
+
+/*
+** This method is the destructor fo a CsvTable object.
+*/
+static int csvtabDisconnect(sqlite3_vtab *pVtab){
+  CsvTable *p = (CsvTable*)pVtab;
+  sqlite3_free(p->zFilename);
+  sqlite3_free(p->zData);
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/* Skip leading whitespace.  Return a pointer to the first non-whitespace
+** character, or to the zero terminator if the string has only whitespace */
+static const char *csv_skip_whitespace(const char *z){
+  while( isspace((unsigned char)z[0]) ) z++;
+  return z;
+}
+
+/* Remove trailing whitespace from the end of string z[] */
+static void csv_trim_whitespace(char *z){
+  size_t n = strlen(z);
+  while( n>0 && isspace((unsigned char)z[n]) ) n--;
+  z[n] = 0;
+}
+
+/* Dequote the string */
+static void csv_dequote(char *z){
+  int i, j;
+  char cQuote = z[0];
+  size_t n;
+
+  if( cQuote!='\'' && cQuote!='"' ) return;
+  n = strlen(z);
+  if( n<2 || z[n-1]!=z[0] ) return;
+  for(i=1, j=0; i<n-1; i++){
+    if( z[i]==cQuote && z[i+1]==cQuote ) i++;
+    z[j++] = z[i];
+  }
+  z[j] = 0;
+}
+
+/* Check to see if the string is of the form:  "TAG = VALUE" with optional
+** whitespace before and around tokens.  If it is, return a pointer to the
+** first character of VALUE.  If it is not, return NULL.
+*/
+static const char *csv_parameter(const char *zTag, int nTag, const char *z){
+  z = csv_skip_whitespace(z);
+  if( strncmp(zTag, z, nTag)!=0 ) return 0;
+  z = csv_skip_whitespace(z+nTag);
+  if( z[0]!='=' ) return 0;
+  return csv_skip_whitespace(z+1);
+}
+
+/* Decode a parameter that requires a dequoted string.
+**
+** Return 1 if the parameter is seen, or 0 if not.  1 is returned
+** even if there is an error.  If an error occurs, then an error message
+** is left in p->zErr.  If there are no errors, p->zErr[0]==0.
+*/
+static int csv_string_parameter(
+  CsvReader *p,            /* Leave the error message here, if there is one */
+  const char *zParam,      /* Parameter we are checking for */
+  const char *zArg,        /* Raw text of the virtual table argment */
+  char **pzVal             /* Write the dequoted string value here */
+){
+  const char *zValue;
+  zValue = csv_parameter(zParam,strlen(zParam),zArg);
+  if( zValue==0 ) return 0;
+  p->zErr[0] = 0;
+  if( *pzVal ){
+    csv_errmsg(p, "more than one '%s' parameter", zParam);
+    return 1;
+  }
+  *pzVal = sqlite3_mprintf("%s", zValue);
+  if( *pzVal==0 ){
+    csv_errmsg(p, "out of memory");
+    return 1;
+  }
+  csv_trim_whitespace(*pzVal);
+  csv_dequote(*pzVal);
+  return 1;
+}
+
+
+/* Return 0 if the argument is false and 1 if it is true.  Return -1 if
+** we cannot really tell.
+*/
+static int csv_boolean(const char *z){
+  if( sqlite3_stricmp("yes",z)==0
+   || sqlite3_stricmp("on",z)==0
+   || sqlite3_stricmp("true",z)==0
+   || (z[0]=='1' && z[0]==0)
+  ){
+    return 1;
+  }
+  if( sqlite3_stricmp("no",z)==0
+   || sqlite3_stricmp("off",z)==0
+   || sqlite3_stricmp("false",z)==0
+   || (z[0]=='0' && z[1]==0)
+  ){
+    return 0;
+  }
+  return -1;
+}
+
+
+/*
+** Parameters:
+**    filename=FILENAME          Name of file containing CSV content
+**    data=TEXT                  Direct CSV content.
+**    schema=SCHEMA              Alternative CSV schema.
+**    header=YES|NO              First row of CSV defines the names of
+**                               columns if "yes".  Default "no".
+**    columns=N                  Assume the CSV file contains N columns.
+**
+** Only available if compiled with SQLITE_TEST:
+**    
+**    testflags=N                Bitmask of test flags.  Optional
+**
+** If schema= is omitted, then the columns are named "c0", "c1", "c2",
+** and so forth.  If columns=N is omitted, then the file is opened and
+** the number of columns in the first row is counted to determine the
+** column count.  If header=YES, then the first row is skipped.
+*/
+static int csvtabConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  CsvTable *pNew = 0;        /* The CsvTable object to construct */
+  int bHeader = -1;          /* header= flags.  -1 means not seen yet */
+  int rc = SQLITE_OK;        /* Result code from this routine */
+  int i, j;                  /* Loop counters */
+#ifdef SQLITE_TEST
+  int tstFlags = 0;          /* Value for testflags=N parameter */
+#endif
+  int nCol = -99;            /* Value of the columns= parameter */
+  CsvReader sRdr;            /* A CSV file reader used to store an error
+                             ** message and/or to count the number of columns */
+  static const char *azParam[] = {
+     "filename", "data", "schema", 
+  };
+  char *azPValue[3];         /* Parameter values */
+# define CSV_FILENAME (azPValue[0])
+# define CSV_DATA     (azPValue[1])
+# define CSV_SCHEMA   (azPValue[2])
+
+
+  assert( sizeof(azPValue)==sizeof(azParam) );
+  memset(&sRdr, 0, sizeof(sRdr));
+  memset(azPValue, 0, sizeof(azPValue));
+  for(i=3; i<argc; i++){
+    const char *z = argv[i];
+    const char *zValue;
+    for(j=0; j<sizeof(azParam)/sizeof(azParam[0]); j++){
+      if( csv_string_parameter(&sRdr, azParam[j], z, &azPValue[j]) ) break;
+    }
+    if( j<sizeof(azParam)/sizeof(azParam[0]) ){
+      if( sRdr.zErr[0] ) goto csvtab_connect_error;
+    }else
+    if( (zValue = csv_parameter("header",6,z))!=0 ){
+      int x;
+      if( bHeader>=0 ){
+        csv_errmsg(&sRdr, "more than one 'header' parameter");
+        goto csvtab_connect_error;
+      }
+      x = csv_boolean(zValue);
+      if( x==1 ){
+        bHeader = 1;
+      }else if( x==0 ){
+        bHeader = 0;
+      }else{
+        csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue);
+        goto csvtab_connect_error;
+      }
+    }else
+#ifdef SQLITE_TEST
+    if( (zValue = csv_parameter("testflags",9,z))!=0 ){
+      tstFlags = (unsigned int)atoi(zValue);
+    }else
+#endif
+    if( (zValue = csv_parameter("columns",7,z))!=0 ){
+      if( nCol>0 ){
+        csv_errmsg(&sRdr, "more than one 'columns' parameter");
+        goto csvtab_connect_error;
+      }
+      nCol = atoi(zValue);
+      if( nCol<=0 ){
+        csv_errmsg(&sRdr, "must have at least one column");
+        goto csvtab_connect_error;
+      }
+    }else
+    {
+      csv_errmsg(&sRdr, "unrecognized parameter '%s'", z);
+      goto csvtab_connect_error;
+    }
+  }
+  if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
+    csv_errmsg(&sRdr, "must either filename= or data= but not both");
+    goto csvtab_connect_error;
+  }
+  if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){
+    goto csvtab_connect_error;
+  }
+  pNew = sqlite3_malloc( sizeof(*pNew) );
+  *ppVtab = (sqlite3_vtab*)pNew;
+  if( pNew==0 ) goto csvtab_connect_oom;
+  memset(pNew, 0, sizeof(*pNew));
+  if( nCol>0 ){
+    pNew->nCol = nCol;
+  }else{
+    do{
+      const char *z = csv_read_one_field(&sRdr);
+      if( z==0 ) goto csvtab_connect_oom;
+      pNew->nCol++;
+    }while( sRdr.cTerm==',' );
+  }
+  pNew->zFilename = CSV_FILENAME;  CSV_FILENAME = 0;
+  pNew->zData = CSV_DATA;          CSV_DATA = 0;
+#ifdef SQLITE_TEST
+  pNew->tstFlags = tstFlags;
+#endif
+  pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0;
+  csv_reader_reset(&sRdr);
+  if( CSV_SCHEMA==0 ){
+    char *zSep = "";
+    CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x(");
+    if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
+    for(i=0; i<pNew->nCol; i++){
+      CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i);
+      zSep = ",";
+    }
+    CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA);
+  }
+  rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
+  if( rc ) goto csvtab_connect_error;
+  for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
+    sqlite3_free(azPValue[i]);
+  }
+  return SQLITE_OK;
+
+csvtab_connect_oom:
+  rc = SQLITE_NOMEM;
+  csv_errmsg(&sRdr, "out of memory");
+
+csvtab_connect_error:
+  if( pNew ) csvtabDisconnect(&pNew->base);
+  for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
+    sqlite3_free(azPValue[i]);
+  }
+  if( sRdr.zErr[0] ){
+    sqlite3_free(*pzErr);
+    *pzErr = sqlite3_mprintf("%s", sRdr.zErr);
+  }
+  csv_reader_reset(&sRdr);
+  if( rc==SQLITE_OK ) rc = SQLITE_ERROR;
+  return rc;
+}
+
+/*
+** Reset the current row content held by a CsvCursor.
+*/
+static void csvtabCursorRowReset(CsvCursor *pCur){
+  CsvTable *pTab = (CsvTable*)pCur->base.pVtab;
+  int i;
+  for(i=0; i<pTab->nCol; i++){
+    sqlite3_free(pCur->azVal[i]);
+    pCur->azVal[i] = 0;
+    pCur->aLen[i] = 0;
+  }
+}
+
+/*
+** The xConnect and xCreate methods do the same thing, but they must be
+** different so that the virtual table is not an eponymous virtual table.
+*/
+static int csvtabCreate(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+ return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr);
+}
+
+/*
+** Destructor for a CsvCursor.
+*/
+static int csvtabClose(sqlite3_vtab_cursor *cur){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  csvtabCursorRowReset(pCur);
+  csv_reader_reset(&pCur->rdr);
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new CsvTable cursor object.
+*/
+static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  CsvTable *pTab = (CsvTable*)p;
+  CsvCursor *pCur;
+  size_t nByte;
+  nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol;
+  pCur = sqlite3_malloc( nByte );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, nByte);
+  pCur->azVal = (char**)&pCur[1];
+  pCur->aLen = (int*)&pCur->azVal[pTab->nCol];
+  *ppCursor = &pCur->base;
+  if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){
+    csv_xfer_error(pTab, &pCur->rdr);
+    return SQLITE_ERROR;
+  }
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a CsvCursor to its next row of input.
+** Set the EOF marker if we reach the end of input.
+*/
+static int csvtabNext(sqlite3_vtab_cursor *cur){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  CsvTable *pTab = (CsvTable*)cur->pVtab;
+  int i = 0;
+  char *z;
+  do{
+    z = csv_read_one_field(&pCur->rdr);
+    if( z==0 ){
+      csv_xfer_error(pTab, &pCur->rdr);
+      break;
+    }
+    if( i<pTab->nCol ){
+      if( pCur->aLen[i] < pCur->rdr.n+1 ){
+        char *zNew = sqlite3_realloc(pCur->azVal[i], pCur->rdr.n+1);
+        if( zNew==0 ){
+          csv_errmsg(&pCur->rdr, "out of memory");
+          csv_xfer_error(pTab, &pCur->rdr);
+          break;
+        }
+        pCur->azVal[i] = zNew;
+        pCur->aLen[i] = pCur->rdr.n+1;
+      }
+      memcpy(pCur->azVal[i], z, pCur->rdr.n+1);
+      i++;
+    }
+  }while( pCur->rdr.cTerm==',' );
+  while( i<pTab->nCol ){
+    sqlite3_free(pCur->azVal[i]);
+    pCur->azVal[i] = 0;
+    pCur->aLen[i] = 0;
+    i++;
+  }
+  if( z==0 || pCur->rdr.cTerm==EOF ){
+    pCur->iRowid = -1;
+  }else{
+    pCur->iRowid++;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the CsvCursor
+** is currently pointing.
+*/
+static int csvtabColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  CsvTable *pTab = (CsvTable*)cur->pVtab;
+  if( i>=0 && i<pTab->nCol && pCur->azVal[i]!=0 ){
+    sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_STATIC);
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.
+*/
+static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int csvtabEof(sqlite3_vtab_cursor *cur){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  return pCur->iRowid<0;
+}
+
+/*
+** Only a full table scan is supported.  So xFilter simply rewinds to
+** the beginning.
+*/
+static int csvtabFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  CsvCursor *pCur = (CsvCursor*)pVtabCursor;
+  CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab;
+  pCur->iRowid = 0;
+  if( pCur->rdr.in==0 ){
+    assert( pCur->rdr.zIn==pTab->zData );
+    assert( pTab->iStart<=pCur->rdr.nIn );
+    pCur->rdr.iIn = pTab->iStart;
+  }else{
+    fseek(pCur->rdr.in, pTab->iStart, SEEK_SET);
+    pCur->rdr.iIn = 0;
+    pCur->rdr.nIn = 0;
+  }
+  return csvtabNext(pVtabCursor);
+}
+
+/*
+** Only a forward full table scan is supported.  xBestIndex is mostly
+** a no-op.  If CSVTEST_FIDX is set, then the presence of equality
+** constraints lowers the estimated cost, which is fiction, but is useful
+** for testing certain kinds of virtual table behavior.
+*/
+static int csvtabBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = 1000000;
+#ifdef SQLITE_TEST
+  if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){
+    /* The usual (and sensible) case is to always do a full table scan.
+    ** The code in this branch only runs when testflags=1.  This code
+    ** generates an artifical and unrealistic plan which is useful
+    ** for testing virtual table logic but is not helpful to real applications.
+    **
+    ** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual
+    ** table (even though it is not) and the cost of running the virtual table
+    ** is reduced from 1 million to just 10.  The constraints are *not* marked
+    ** as omittable, however, so the query planner should still generate a
+    ** plan that gives a correct answer, even if they plan is not optimal.
+    */
+    int i;
+    int nConst = 0;
+    for(i=0; i<pIdxInfo->nConstraint; i++){
+      unsigned char op;
+      if( pIdxInfo->aConstraint[i].usable==0 ) continue;
+      op = pIdxInfo->aConstraint[i].op;
+      if( op==SQLITE_INDEX_CONSTRAINT_EQ 
+       || op==SQLITE_INDEX_CONSTRAINT_LIKE
+       || op==SQLITE_INDEX_CONSTRAINT_GLOB
+      ){
+        pIdxInfo->estimatedCost = 10;
+        pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1;
+        nConst++;
+      }
+    }
+  }
+#endif
+  return SQLITE_OK;
+}
+
+
+static sqlite3_module CsvModule = {
+  0,                       /* iVersion */
+  csvtabCreate,            /* xCreate */
+  csvtabConnect,           /* xConnect */
+  csvtabBestIndex,         /* xBestIndex */
+  csvtabDisconnect,        /* xDisconnect */
+  csvtabDisconnect,        /* xDestroy */
+  csvtabOpen,              /* xOpen - open a cursor */
+  csvtabClose,             /* xClose - close a cursor */
+  csvtabFilter,            /* xFilter - configure scan constraints */
+  csvtabNext,              /* xNext - advance a cursor */
+  csvtabEof,               /* xEof - check for end of scan */
+  csvtabColumn,            /* xColumn - read data */
+  csvtabRowid,             /* xRowid - read data */
+  0,                       /* xUpdate */
+  0,                       /* xBegin */
+  0,                       /* xSync */
+  0,                       /* xCommit */
+  0,                       /* xRollback */
+  0,                       /* xFindMethod */
+  0,                       /* xRename */
+};
+
+#ifdef SQLITE_TEST
+/*
+** For virtual table testing, make a version of the CSV virtual table
+** available that has an xUpdate function.  But the xUpdate always returns
+** SQLITE_READONLY since the CSV file is not really writable.
+*/
+static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){
+  return SQLITE_READONLY;
+}
+static sqlite3_module CsvModuleFauxWrite = {
+  0,                       /* iVersion */
+  csvtabCreate,            /* xCreate */
+  csvtabConnect,           /* xConnect */
+  csvtabBestIndex,         /* xBestIndex */
+  csvtabDisconnect,        /* xDisconnect */
+  csvtabDisconnect,        /* xDestroy */
+  csvtabOpen,              /* xOpen - open a cursor */
+  csvtabClose,             /* xClose - close a cursor */
+  csvtabFilter,            /* xFilter - configure scan constraints */
+  csvtabNext,              /* xNext - advance a cursor */
+  csvtabEof,               /* xEof - check for end of scan */
+  csvtabColumn,            /* xColumn - read data */
+  csvtabRowid,             /* xRowid - read data */
+  csvtabUpdate,            /* xUpdate */
+  0,                       /* xBegin */
+  0,                       /* xSync */
+  0,                       /* xCommit */
+  0,                       /* xRollback */
+  0,                       /* xFindMethod */
+  0,                       /* xRename */
+};
+#endif /* SQLITE_TEST */
+
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+/* 
+** This routine is called when the extension is loaded.  The new
+** CSV virtual table module is registered with the calling database
+** connection.
+*/
+int sqlite3_csv_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc;
+  SQLITE_EXTENSION_INIT2(pApi);
+  rc = sqlite3_create_module(db, "csv", &CsvModule, 0);
+#ifdef SQLITE_TEST
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0);
+  }
+#endif
+  return rc;
+}

ext/misc/scrub.c

@@ -0,0 +1,594 @@
+/*
+** 2016-05-05
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file implements a utility function (and a utility program) that
+** makes a copy of an SQLite database while simultaneously zeroing out all
+** deleted content.
+**
+** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite
+** deletes content, it does not overwrite the deleted content but rather marks
+** the region of the file that held that content as being reusable.  This can
+** cause deleted content to recoverable from the database file.  This stale
+** content is removed by the VACUUM command, but VACUUM can be expensive for
+** large databases.  When in PRAGMA secure_delete=ON mode, the deleted content
+** is zeroed, but secure_delete=ON has overhead as well.
+**
+** This utility attempts to make a copy of a complete SQLite database where
+** all of the deleted content is zeroed out in the copy, and it attempts to
+** do so while being faster than running VACUUM.
+**
+** Usage:
+**
+**   int sqlite3_scrub_backup(
+**       const char *zSourceFile,   // Source database filename
+**       const char *zDestFile,     // Destination database filename
+**       char **pzErrMsg            // Write error message here
+**   );
+**
+** Simply call the API above specifying the filename of the source database
+** and the name of the backup copy.  The source database must already exist
+** and can be in active use. (A read lock is held during the backup.)  The
+** destination file should not previously exist.  If the pzErrMsg parameter
+** is non-NULL and if an error occurs, then an error message might be written
+** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to
+** that error message.  But if the error is an OOM, the error might not be
+** reported.  The routine always returns non-zero if there is an error.
+**
+** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and
+** this file becomes a standalone program that can be run as follows:
+**
+**      ./sqlite3scrub SOURCE DEST
+*/
+#include "sqlite3.h"
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+
+typedef struct ScrubState ScrubState;
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+
+
+/* State information for a scrub-and-backup operation */
+struct ScrubState {
+  const char *zSrcFile;    /* Name of the source file */
+  const char *zDestFile;   /* Name of the destination file */
+  int rcErr;               /* Error code */
+  char *zErr;              /* Error message text */
+  sqlite3 *dbSrc;          /* Source database connection */
+  sqlite3_file *pSrc;      /* Source file handle */
+  sqlite3 *dbDest;         /* Destination database connection */
+  sqlite3_file *pDest;     /* Destination file handle */
+  u32 szPage;              /* Page size */
+  u32 szUsable;            /* Usable bytes on each page */
+  u32 nPage;               /* Number of pages */
+  u8 *page1;               /* Content of page 1 */
+};
+
+/* Store an error message */
+static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){
+  va_list ap;
+  sqlite3_free(p->zErr);
+  va_start(ap, zFormat);
+  p->zErr = sqlite3_vmprintf(zFormat, ap);
+  va_end(ap);
+  if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR;
+}
+
+/* Allocate memory to hold a single page of content */
+static u8 *scrubBackupAllocPage(ScrubState *p){
+  u8 *pPage;
+  if( p->rcErr ) return 0;
+  pPage = sqlite3_malloc( p->szPage );
+  if( pPage==0 ) p->rcErr = SQLITE_NOMEM;
+  return pPage;
+}
+
+/* Read a page from the source database into memory.  Use the memory
+** provided by pBuf if not NULL or allocate a new page if pBuf==NULL.
+*/
+static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){
+  int rc;
+  sqlite3_int64 iOff;
+  u8 *pOut = pBuf;
+  if( p->rcErr ) return 0;
+  if( pOut==0 ){
+    pOut = scrubBackupAllocPage(p);
+    if( pOut==0 ) return 0;
+  }
+  iOff = (pgno-1)*(sqlite3_int64)p->szPage;
+  rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff);
+  if( rc!=SQLITE_OK ){
+    if( pBuf==0 ) sqlite3_free(pOut);
+    pOut = 0;
+    scrubBackupErr(p, "read failed for page %d", pgno);
+    p->rcErr = SQLITE_IOERR;
+  }
+  return pOut;  
+}
+
+/* Write a page to the destination database */
+static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){
+  int rc;
+  sqlite3_int64 iOff;
+  if( p->rcErr ) return;
+  iOff = (pgno-1)*(sqlite3_int64)p->szPage;
+  rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff);
+  if( rc!=SQLITE_OK ){
+    scrubBackupErr(p, "write failed for page %d", pgno);
+    p->rcErr = SQLITE_IOERR;
+  }
+}
+
+/* Prepare a statement against the "db" database. */
+static sqlite3_stmt *scrubBackupPrepare(
+  ScrubState *p,      /* Backup context */
+  sqlite3 *db,        /* Database to prepare against */
+  const char *zSql    /* SQL statement */
+){
+  sqlite3_stmt *pStmt;
+  if( p->rcErr ) return 0;
+  p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  if( p->rcErr ){
+    scrubBackupErr(p, "SQL error \"%s\" on \"%s\"",
+                   sqlite3_errmsg(db), zSql);
+    sqlite3_finalize(pStmt);
+    return 0;
+  }
+  return pStmt;
+}
+
+
+/* Open the source database file */
+static void scrubBackupOpenSrc(ScrubState *p){
+  sqlite3_stmt *pStmt;
+  int rc;
+  /* Open the source database file */
+  p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc,
+                 SQLITE_OPEN_READWRITE |
+                 SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
+  if( p->rcErr ){
+    scrubBackupErr(p, "cannot open source database: %s",
+                      sqlite3_errmsg(p->dbSrc));
+    return;
+  }
+  p->rcErr = sqlite3_exec(p->dbSrc, "SELECT 1 FROM sqlite_master; BEGIN;",
+                          0, 0, 0);
+  if( p->rcErr ){
+    scrubBackupErr(p,
+       "cannot start a read transaction on the source database: %s",
+       sqlite3_errmsg(p->dbSrc));
+    return;
+  }
+  rc = sqlite3_wal_checkpoint_v2(p->dbSrc, "main", SQLITE_CHECKPOINT_FULL,
+                                 0, 0);
+  if( rc ){
+    scrubBackupErr(p, "cannot checkpoint the source database");
+    return;
+  }
+  pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size");
+  if( pStmt==0 ) return;
+  rc = sqlite3_step(pStmt);
+  if( rc==SQLITE_ROW ){
+    p->szPage = sqlite3_column_int(pStmt, 0);
+  }else{
+    scrubBackupErr(p, "unable to determine the page size");
+  }
+  sqlite3_finalize(pStmt);
+  if( p->rcErr ) return;
+  pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count");
+  if( pStmt==0 ) return;
+  rc = sqlite3_step(pStmt);
+  if( rc==SQLITE_ROW ){
+    p->nPage = sqlite3_column_int(pStmt, 0);
+  }else{
+    scrubBackupErr(p, "unable to determine the size of the source database");
+  }
+  sqlite3_finalize(pStmt);
+  sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc);
+  if( p->pSrc==0 || p->pSrc->pMethods==0 ){
+    scrubBackupErr(p, "cannot get the source file handle");
+    p->rcErr = SQLITE_ERROR;
+  }
+}
+
+/* Create and open the destination file */
+static void scrubBackupOpenDest(ScrubState *p){
+  sqlite3_stmt *pStmt;
+  int rc;
+  char *zSql;
+  if( p->rcErr ) return;
+  p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest,
+                 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
+                 SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
+  if( p->rcErr ){
+    scrubBackupErr(p, "cannot open destination database: %s",
+                      sqlite3_errmsg(p->dbDest));
+    return;
+  }
+  zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage);
+  if( zSql==0 ){
+    p->rcErr = SQLITE_NOMEM;
+    return;
+  }
+  p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0);
+  sqlite3_free(zSql);
+  if( p->rcErr ){
+    scrubBackupErr(p,
+       "cannot set the page size on the destination database: %s",
+       sqlite3_errmsg(p->dbDest));
+    return;
+  }
+  sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0);
+  p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0);
+  if( p->rcErr ){
+    scrubBackupErr(p,
+       "cannot start a write transaction on the destination database: %s",
+       sqlite3_errmsg(p->dbDest));
+    return;
+  }
+  pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;");
+  if( pStmt==0 ) return;
+  rc = sqlite3_step(pStmt);
+  if( rc!=SQLITE_ROW ){
+    scrubBackupErr(p, "cannot measure the size of the destination");
+  }else if( sqlite3_column_int(pStmt, 0)>1 ){
+    scrubBackupErr(p, "destination database is not empty - holds %d pages",
+                   sqlite3_column_int(pStmt, 0));
+  }
+  sqlite3_finalize(pStmt);
+  sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest);
+  if( p->pDest==0 || p->pDest->pMethods==0 ){
+    scrubBackupErr(p, "cannot get the destination file handle");
+    p->rcErr = SQLITE_ERROR;
+  }
+}
+
+/* Read a 32-bit big-endian integer */
+static u32 scrubBackupInt32(const u8 *a){
+  u32 v = a[3];
+  v += ((u32)a[2])<<8;
+  v += ((u32)a[1])<<16;
+  v += ((u32)a[0])<<24;
+  return v;
+}
+
+/* Read a 16-bit big-endian integer */
+static u32 scrubBackupInt16(const u8 *a){
+  return (a[0]<<8) + a[1];
+}
+
+/*
+** Read a varint.  Put the value in *pVal and return the number of bytes.
+*/
+static int scrubBackupVarint(const u8 *z, sqlite3_int64 *pVal){
+  sqlite3_int64 v = 0;
+  int i;
+  for(i=0; i<8; i++){
+    v = (v<<7) + (z[i]&0x7f);
+    if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; }
+  }
+  v = (v<<8) + (z[i]&0xff);
+  *pVal = v;
+  return 9;
+}
+
+/*
+** Return the number of bytes in a varint.
+*/
+static int scrubBackupVarintSize(const u8 *z){
+  int i;
+  for(i=0; i<8; i++){
+    if( (z[i]&0x80)==0 ){ return i+1; }
+  }
+  return 9;
+}
+
+/*
+** Copy the freelist trunk page given, and all its descendents,
+** zeroing out as much as possible in the process.
+*/
+static void scrubBackupFreelist(ScrubState *p, int pgno, u32 nFree){
+  u8 *a, *aBuf;
+  u32 n, mx;
+
+  if( p->rcErr ) return;
+  aBuf = scrubBackupAllocPage(p);
+  if( aBuf==0 ) return;
+ 
+  while( pgno && nFree){
+    a = scrubBackupRead(p, pgno, aBuf);
+    if( a==0 ) break;
+    n = scrubBackupInt32(&a[4]);
+    mx = p->szUsable/4 - 2;
+    if( n<mx ){
+      memset(&a[n*4+8], 0, 4*(mx-n));
+    }
+    scrubBackupWrite(p, pgno, a);
+    pgno = scrubBackupInt32(a);
+#if 0
+    /* There is really no point in copying the freelist leaf pages.
+    ** Simply leave them uninitialized in the destination database.  The
+    ** OS filesystem should zero those pages for us automatically.
+    */
+    for(i=0; i<n && nFree; i++){
+      u32 iLeaf = scrubBackupInt32(&a[i*4+8]);
+      if( aZero==0 ){
+        aZero = scrubBackupAllocPage(p);
+        if( aZero==0 ){ pgno = 0; break; }
+        memset(aZero, 0, p->szPage);
+      }
+      scrubBackupWrite(p, iLeaf, aZero);
+      nFree--;
+    }
+#endif
+  }
+  sqlite3_free(aBuf);
+}
+
+/*
+** Copy an overflow chain from source to destination.  Zero out any
+** unused tail at the end of the overflow chain.
+*/
+static void scrubBackupOverflow(ScrubState *p, int pgno, u32 nByte){
+  u8 *a, *aBuf;
+
+  aBuf = scrubBackupAllocPage(p);
+  if( aBuf==0 ) return;
+  while( nByte>0 && pgno!=0 ){
+    a = scrubBackupRead(p, pgno, aBuf);
+    if( a==0 ) break;
+    if( nByte >= (p->szUsable)-4 ){
+      nByte -= (p->szUsable) - 4;
+    }else{
+      u32 x = (p->szUsable - 4) - nByte;
+      u32 i = p->szUsable - x;
+      memset(&a[i], 0, x);
+      nByte = 0;
+    }
+    scrubBackupWrite(p, pgno, a);
+    pgno = scrubBackupInt32(a);
+  }
+  sqlite3_free(aBuf);      
+}
+   
+
+/*
+** Copy B-Tree page pgno, and all of its children, from source to destination.
+** Zero out deleted content during the copy.
+*/
+static void scrubBackupBtree(ScrubState *p, int pgno, int iDepth){
+  u8 *a;
+  u32 i, n, pc;
+  u32 nCell;
+  u32 nPrefix;
+  u32 szHdr;
+  u32 iChild;
+  u8 *aTop;
+  u8 *aCell;
+  u32 x, y;
+  int ln = 0;
+
+  
+  if( p->rcErr ) return;
+  if( iDepth>50 ){
+    scrubBackupErr(p, "corrupt: b-tree too deep at page %d", pgno);
+    return;
+  }
+  if( pgno==1 ){
+    a = p->page1;
+  }else{
+    a = scrubBackupRead(p, pgno, 0);
+    if( a==0 ) return;
+  }
+  nPrefix = pgno==1 ? 100 : 0;
+  aTop = &a[nPrefix];
+  szHdr = 8 + 4*(aTop[0]==0x02 || aTop[0]==0x05);
+  aCell = aTop + szHdr;
+  nCell = scrubBackupInt16(&aTop[3]);
+
+  /* Zero out the gap between the cell index and the start of the
+  ** cell content area */
+  x = scrubBackupInt16(&aTop[5]);  /* First byte of cell content area */
+  if( x>p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
+  y = szHdr + nPrefix + nCell*2;
+  if( y>x ){ ln=__LINE__; goto btree_corrupt; }
+  if( y<x ) memset(a+y, 0, x-y);  /* Zero the gap */
+
+  /* Zero out all the free blocks */  
+  pc = scrubBackupInt16(&aTop[1]);
+  if( pc>0 && pc<x ){ ln=__LINE__; goto btree_corrupt; }
+  while( pc ){
+    if( pc>(p->szUsable)-4 ){ ln=__LINE__; goto btree_corrupt; }
+    n = scrubBackupInt16(&a[pc+2]);
+    if( pc+n>(p->szUsable) ){ ln=__LINE__; goto btree_corrupt; }
+    if( n>4 ) memset(&a[pc+4], 0, n-4);
+    x = scrubBackupInt16(&a[pc]);
+    if( x<pc+4 && x>0 ){ ln=__LINE__; goto btree_corrupt; }
+    pc = x;
+  }
+
+  /* Write this one page */
+  scrubBackupWrite(p, pgno, a);
+
+  /* Walk the tree and process child pages */
+  for(i=0; i<nCell; i++){
+    u32 X, M, K, nLocal;
+    sqlite3_int64 P;
+    pc = scrubBackupInt16(&aCell[i*2]);
+    if( pc <= szHdr ){ ln=__LINE__; goto btree_corrupt; }
+    if( pc > p->szUsable-3 ){ ln=__LINE__; goto btree_corrupt; }
+    if( aTop[0]==0x05 || aTop[0]==0x02 ){
+      if( pc+4 > p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
+      iChild = scrubBackupInt32(&a[pc]);
+      pc += 4;
+      scrubBackupBtree(p, iChild, iDepth+1);
+      if( aTop[0]==0x05 ) continue;
+    }
+    pc += scrubBackupVarint(&a[pc], &P);
+    if( pc >= p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
+    if( aTop[0]==0x0d ){
+      X = p->szUsable - 35;
+    }else{
+      X = ((p->szUsable - 12)*64/255) - 23;
+    }
+    if( P<=X ){
+      /* All content is local.  No overflow */
+      continue;
+    }
+    M = ((p->szUsable - 12)*32/255)-23;
+    K = M + ((P-M)%(p->szUsable-4));
+    if( aTop[0]==0x0d ){
+      pc += scrubBackupVarintSize(&a[pc]);
+      if( pc > (p->szUsable-4) ){ ln=__LINE__; goto btree_corrupt; }
+    }
+    nLocal = K<=X ? K : M;
+    if( pc+nLocal > p->szUsable-4 ){ ln=__LINE__; goto btree_corrupt; }
+    iChild = scrubBackupInt32(&a[pc+nLocal]);
+    scrubBackupOverflow(p, iChild, P-nLocal);
+  }
+
+  /* Walk the right-most tree */
+  if( aTop[0]==0x05 || aTop[0]==0x02 ){
+    iChild = scrubBackupInt32(&aTop[8]);
+    scrubBackupBtree(p, iChild, iDepth+1);
+  }
+
+  /* All done */
+  if( pgno>1 ) sqlite3_free(a);
+  return;
+
+btree_corrupt:
+  scrubBackupErr(p, "corruption on page %d of source database (errid=%d)",
+                 pgno, ln);
+  if( pgno>1 ) sqlite3_free(a);  
+}
+
+/*
+** Copy all ptrmap pages from source to destination.
+** This routine is only called if the source database is in autovacuum
+** or incremental vacuum mode.
+*/
+static void scrubBackupPtrmap(ScrubState *p){
+  u32 pgno = 2;
+  u32 J = p->szUsable/5;
+  u32 iLock = (1073742335/p->szPage)+1;
+  u8 *a, *pBuf;
+  if( p->rcErr ) return;
+  pBuf = scrubBackupAllocPage(p);
+  if( pBuf==0 ) return;
+  while( pgno<=p->nPage ){
+    a = scrubBackupRead(p, pgno, pBuf);
+    if( a==0 ) break;
+    scrubBackupWrite(p, pgno, a);
+    pgno += J+1;
+    if( pgno==iLock ) pgno++;
+  }
+  sqlite3_free(pBuf);
+}
+
+int sqlite3_scrub_backup(
+  const char *zSrcFile,    /* Source file */
+  const char *zDestFile,   /* Destination file */
+  char **pzErr             /* Write error here if non-NULL */
+){
+  ScrubState s;
+  u32 n, i;
+  sqlite3_stmt *pStmt;
+
+  memset(&s, 0, sizeof(s));
+  s.zSrcFile = zSrcFile;
+  s.zDestFile = zDestFile;
+
+  /* Open both source and destination databases */
+  scrubBackupOpenSrc(&s);
+  scrubBackupOpenDest(&s);
+
+  /* Read in page 1 */
+  s.page1 = scrubBackupRead(&s, 1, 0);
+  if( s.page1==0 ) goto scrub_abort;
+  s.szUsable = s.szPage - s.page1[20];
+
+  /* Copy the freelist */    
+  n = scrubBackupInt32(&s.page1[36]);
+  i = scrubBackupInt32(&s.page1[32]);
+  if( n ) scrubBackupFreelist(&s, i, n);
+
+  /* Copy ptrmap pages */
+  n = scrubBackupInt32(&s.page1[52]);
+  if( n ) scrubBackupPtrmap(&s);
+
+  /* Copy all of the btrees */
+  scrubBackupBtree(&s, 1, 0);
+  pStmt = scrubBackupPrepare(&s, s.dbSrc,
+       "SELECT rootpage FROM sqlite_master WHERE coalesce(rootpage,0)>0");
+  if( pStmt==0 ) goto scrub_abort;
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    i = (u32)sqlite3_column_int(pStmt, 0);
+    scrubBackupBtree(&s, i, 0);
+  }
+  sqlite3_finalize(pStmt);
+
+scrub_abort:    
+  /* Close the destination database without closing the transaction. If we
+  ** commit, page zero will be overwritten. */
+  sqlite3_close(s.dbDest);
+
+  /* But do close out the read-transaction on the source database */
+  sqlite3_exec(s.dbSrc, "COMMIT;", 0, 0, 0);
+  sqlite3_close(s.dbSrc);
+  sqlite3_free(s.page1);
+  if( pzErr ){
+    *pzErr = s.zErr;
+  }else{
+    sqlite3_free(s.zErr);
+  }
+  return s.rcErr;
+}   
+
+#ifdef SCRUB_STANDALONE
+/* Error and warning log */
+static void errorLogCallback(void *pNotUsed, int iErr, const char *zMsg){
+  const char *zType;
+  switch( iErr&0xff ){
+    case SQLITE_WARNING: zType = "WARNING";  break;
+    case SQLITE_NOTICE:  zType = "NOTICE";   break;
+    default:             zType = "ERROR";    break;
+  }
+  fprintf(stderr, "%s: %s\n", zType, zMsg);
+}
+
+/* The main() routine when this utility is run as a stand-alone program */
+int main(int argc, char **argv){
+  char *zErr = 0;
+  int rc;
+  if( argc!=3 ){
+    fprintf(stderr,"Usage: %s SOURCE DESTINATION\n", argv[0]);
+    exit(1);
+  }
+  sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, 0);
+  rc = sqlite3_scrub_backup(argv[1], argv[2], &zErr);
+  if( rc==SQLITE_NOMEM ){
+    fprintf(stderr, "%s: out of memory\n", argv[0]);
+    exit(1);
+  }
+  if( zErr ){
+    fprintf(stderr, "%s: %s\n", argv[0], zErr);
+    sqlite3_free(zErr);
+    exit(1);
+  }
+  return 0;
+}
+#endif

ext/misc/vfsstat.c

@@ -0,0 +1,816 @@
+/*
+** 2016-05-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains the implementation of an SQLite vfs shim that
+** tracks I/O.  Access to the accumulated status counts is provided using
+** an eponymous virtual table.
+*/
+#include <sqlite3ext.h>
+SQLITE_EXTENSION_INIT1
+
+/*
+** This module contains code for a wrapper VFS that cause stats for
+** most VFS calls to be recorded.
+**
+** To use this module, first compile it as a loadable extension.  See
+** https://www.sqlite.org/loadext.html#build for compilations instructions.
+**
+** After compliing, load this extension, then open database connections to be
+** measured.  Query usages status using the vfsstat virtual table:
+**
+**         SELECT * FROM vfsstat;
+**
+** Reset counters using UPDATE statements against vfsstat:
+**
+**         UPDATE vfsstat SET count=0;
+**
+** EXAMPLE SCRIPT:
+**
+**      .load ./vfsstat
+**      .open test.db
+**      DROP TABLE IF EXISTS t1;
+**      CREATE TABLE t1(x,y);
+**      INSERT INTO t1 VALUES(123, randomblob(5000));
+**      CREATE INDEX t1x ON t1(x);
+**      DROP TABLE t1;
+**      VACUUM;
+**      SELECT * FROM vfsstat WHERE count>0;
+**
+** LIMITATIONS:
+** 
+** This module increments counters without using mutex protection.  So if
+** two or more threads try to use this module at the same time, race conditions
+** may occur which mess up the counts.  This is harmless, other than giving
+** incorrect statistics.
+*/
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+
+/*
+** File types
+*/
+#define VFSSTAT_MAIN         0   /* Main database file */
+#define VFSSTAT_JOURNAL      1   /* Rollback journal */
+#define VFSSTAT_WAL          2   /* Write-ahead log file */
+#define VFSSTAT_MASTERJRNL   3   /* Master journal */
+#define VFSSTAT_SUBJRNL      4   /* Subjournal */
+#define VFSSTAT_TEMPDB       5   /* TEMP database */
+#define VFSSTAT_TEMPJRNL     6   /* Journal for TEMP database */
+#define VFSSTAT_TRANSIENT    7   /* Transient database */
+#define VFSSTAT_ANY          8   /* Unspecified file type */
+#define VFSSTAT_nFile        9   /* This many file types */
+
+/* Names of the file types.  These are allowed values for the
+** first column of the vfsstat virtual table.
+*/
+static const char *azFile[] = {
+  "database", "journal", "wal", "master-journal", "sub-journal",
+  "temp-database", "temp-journal", "transient-db", "*"
+};
+
+/*
+** Stat types
+*/
+#define VFSSTAT_BYTESIN      0   /* Bytes read in */
+#define VFSSTAT_BYTESOUT     1   /* Bytes written out */   
+#define VFSSTAT_READ         2   /* Read requests */
+#define VFSSTAT_WRITE        3   /* Write requests */
+#define VFSSTAT_SYNC         4   /* Syncs */
+#define VFSSTAT_OPEN         5   /* File opens */
+#define VFSSTAT_LOCK         6   /* Lock requests */
+#define VFSSTAT_ACCESS       0   /* xAccess calls.  filetype==ANY only */
+#define VFSSTAT_DELETE       1   /* xDelete calls.  filetype==ANY only */
+#define VFSSTAT_FULLPATH     2   /* xFullPathname calls.  ANY only */
+#define VFSSTAT_RANDOM       3   /* xRandomness calls.    ANY only */
+#define VFSSTAT_SLEEP        4   /* xSleep calls.         ANY only */
+#define VFSSTAT_CURTIME      5   /* xCurrentTime calls.   ANY only */
+#define VFSSTAT_nStat        7   /* This many stat types */
+
+
+/* Names for the second column of the vfsstat virtual table for all
+** cases except when the first column is "*" or VFSSTAT_ANY. */
+static const char *azStat[] = {
+  "bytes-in", "bytes-out", "read", "write", "sync", "open", "lock",
+};
+static const char *azStatAny[] = {
+  "access", "delete", "fullpathname", "randomness", "sleep", "currenttimestamp",
+  "not-used"
+};
+
+/* Total number of counters */
+#define VFSSTAT_MXCNT  (VFSSTAT_nStat*VFSSTAT_nFile)
+
+/*
+** Performance stats are collected in an instance of the following
+** global array.
+*/
+static sqlite3_uint64 aVfsCnt[VFSSTAT_MXCNT];
+
+/*
+** Access to a specific counter
+*/
+#define STATCNT(filetype,stat) (aVfsCnt[(filetype)*VFSSTAT_nStat+(stat)])
+
+/*
+** Forward declaration of objects used by this utility
+*/
+typedef struct VStatVfs VStatVfs;
+typedef struct VStatFile VStatFile;
+
+/* An instance of the VFS */
+struct VStatVfs {
+  sqlite3_vfs base;               /* VFS methods */
+  sqlite3_vfs *pVfs;              /* Parent VFS */
+};
+
+/* An open file */
+struct VStatFile {
+  sqlite3_file base;              /* IO methods */
+  sqlite3_file *pReal;            /* Underlying file handle */
+  unsigned char eFiletype;        /* What type of file is this */
+};
+
+#define REALVFS(p) (((VStatVfs*)(p))->pVfs)
+
+/*
+** Methods for VStatFile
+*/
+static int vstatClose(sqlite3_file*);
+static int vstatRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int vstatWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
+static int vstatTruncate(sqlite3_file*, sqlite3_int64 size);
+static int vstatSync(sqlite3_file*, int flags);
+static int vstatFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int vstatLock(sqlite3_file*, int);
+static int vstatUnlock(sqlite3_file*, int);
+static int vstatCheckReservedLock(sqlite3_file*, int *pResOut);
+static int vstatFileControl(sqlite3_file*, int op, void *pArg);
+static int vstatSectorSize(sqlite3_file*);
+static int vstatDeviceCharacteristics(sqlite3_file*);
+static int vstatShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
+static int vstatShmLock(sqlite3_file*, int offset, int n, int flags);
+static void vstatShmBarrier(sqlite3_file*);
+static int vstatShmUnmap(sqlite3_file*, int deleteFlag);
+static int vstatFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
+static int vstatUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
+
+/*
+** Methods for VStatVfs
+*/
+static int vstatOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
+static int vstatDelete(sqlite3_vfs*, const char *zName, int syncDir);
+static int vstatAccess(sqlite3_vfs*, const char *zName, int flags, int *);
+static int vstatFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
+static void *vstatDlOpen(sqlite3_vfs*, const char *zFilename);
+static void vstatDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
+static void (*vstatDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
+static void vstatDlClose(sqlite3_vfs*, void*);
+static int vstatRandomness(sqlite3_vfs*, int nByte, char *zOut);
+static int vstatSleep(sqlite3_vfs*, int microseconds);
+static int vstatCurrentTime(sqlite3_vfs*, double*);
+static int vstatGetLastError(sqlite3_vfs*, int, char *);
+static int vstatCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+
+static VStatVfs vstat_vfs = {
+  {
+    2,                            /* iVersion */
+    0,                            /* szOsFile (set by register_vstat()) */
+    1024,                         /* mxPathname */
+    0,                            /* pNext */
+    "vfslog",                     /* zName */
+    0,                            /* pAppData */
+    vstatOpen,                     /* xOpen */
+    vstatDelete,                   /* xDelete */
+    vstatAccess,                   /* xAccess */
+    vstatFullPathname,             /* xFullPathname */
+    vstatDlOpen,                   /* xDlOpen */
+    vstatDlError,                  /* xDlError */
+    vstatDlSym,                    /* xDlSym */
+    vstatDlClose,                  /* xDlClose */
+    vstatRandomness,               /* xRandomness */
+    vstatSleep,                    /* xSleep */
+    vstatCurrentTime,              /* xCurrentTime */
+    vstatGetLastError,             /* xGetLastError */
+    vstatCurrentTimeInt64          /* xCurrentTimeInt64 */
+  },
+  0
+};
+
+static const sqlite3_io_methods vstat_io_methods = {
+  3,                              /* iVersion */
+  vstatClose,                      /* xClose */
+  vstatRead,                       /* xRead */
+  vstatWrite,                      /* xWrite */
+  vstatTruncate,                   /* xTruncate */
+  vstatSync,                       /* xSync */
+  vstatFileSize,                   /* xFileSize */
+  vstatLock,                       /* xLock */
+  vstatUnlock,                     /* xUnlock */
+  vstatCheckReservedLock,          /* xCheckReservedLock */
+  vstatFileControl,                /* xFileControl */
+  vstatSectorSize,                 /* xSectorSize */
+  vstatDeviceCharacteristics,      /* xDeviceCharacteristics */
+  vstatShmMap,                     /* xShmMap */
+  vstatShmLock,                    /* xShmLock */
+  vstatShmBarrier,                 /* xShmBarrier */
+  vstatShmUnmap,                   /* xShmUnmap */
+  vstatFetch,                      /* xFetch */
+  vstatUnfetch                     /* xUnfetch */
+};
+
+
+
+/*
+** Close an vstat-file.
+*/
+static int vstatClose(sqlite3_file *pFile){
+  VStatFile *p = (VStatFile *)pFile;
+  int rc = SQLITE_OK;
+
+  if( p->pReal->pMethods ){
+    rc = p->pReal->pMethods->xClose(p->pReal);
+  }
+  return rc;
+}
+
+
+/*
+** Read data from an vstat-file.
+*/
+static int vstatRead(
+  sqlite3_file *pFile, 
+  void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+
+  rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
+  STATCNT(p->eFiletype,VFSSTAT_READ)++;
+  if( rc==SQLITE_OK ){
+    STATCNT(p->eFiletype,VFSSTAT_BYTESIN) += iAmt;
+  }
+  return rc;
+}
+
+/*
+** Write data to an vstat-file.
+*/
+static int vstatWrite(
+  sqlite3_file *pFile,
+  const void *z,
+  int iAmt,
+  sqlite_int64 iOfst
+){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+
+  rc = p->pReal->pMethods->xWrite(p->pReal, z, iAmt, iOfst);
+  STATCNT(p->eFiletype,VFSSTAT_WRITE)++;
+  if( rc==SQLITE_OK ){
+    STATCNT(p->eFiletype,VFSSTAT_BYTESOUT) += iAmt;
+  }
+  return rc;
+}
+
+/*
+** Truncate an vstat-file.
+*/
+static int vstatTruncate(sqlite3_file *pFile, sqlite_int64 size){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xTruncate(p->pReal, size);
+  return rc;
+}
+
+/*
+** Sync an vstat-file.
+*/
+static int vstatSync(sqlite3_file *pFile, int flags){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xSync(p->pReal, flags);
+  STATCNT(p->eFiletype,VFSSTAT_SYNC)++;
+  return rc;
+}
+
+/*
+** Return the current file-size of an vstat-file.
+*/
+static int vstatFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
+  return rc;
+}
+
+/*
+** Lock an vstat-file.
+*/
+static int vstatLock(sqlite3_file *pFile, int eLock){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xLock(p->pReal, eLock);
+  STATCNT(p->eFiletype,VFSSTAT_LOCK)++;
+  return rc;
+}
+
+/*
+** Unlock an vstat-file.
+*/
+static int vstatUnlock(sqlite3_file *pFile, int eLock){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xUnlock(p->pReal, eLock);
+  STATCNT(p->eFiletype,VFSSTAT_LOCK)++;
+  return rc;
+}
+
+/*
+** Check if another file-handle holds a RESERVED lock on an vstat-file.
+*/
+static int vstatCheckReservedLock(sqlite3_file *pFile, int *pResOut){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut);
+  STATCNT(p->eFiletype,VFSSTAT_LOCK)++;
+  return rc;
+}
+
+/*
+** File control method. For custom operations on an vstat-file.
+*/
+static int vstatFileControl(sqlite3_file *pFile, int op, void *pArg){
+  VStatFile *p = (VStatFile *)pFile;
+  int rc;
+  rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg);
+  if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){
+    *(char**)pArg = sqlite3_mprintf("vstat/%z", *(char**)pArg);
+  }
+  return rc;
+}
+
+/*
+** Return the sector-size in bytes for an vstat-file.
+*/
+static int vstatSectorSize(sqlite3_file *pFile){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xSectorSize(p->pReal);
+  return rc;
+}
+
+/*
+** Return the device characteristic flags supported by an vstat-file.
+*/
+static int vstatDeviceCharacteristics(sqlite3_file *pFile){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal);
+  return rc;
+}
+
+/* Create a shared memory file mapping */
+static int vstatShmMap(
+  sqlite3_file *pFile,
+  int iPg,
+  int pgsz,
+  int bExtend,
+  void volatile **pp
+){
+  VStatFile *p = (VStatFile *)pFile;
+  return p->pReal->pMethods->xShmMap(p->pReal, iPg, pgsz, bExtend, pp);
+}
+
+/* Perform locking on a shared-memory segment */
+static int vstatShmLock(sqlite3_file *pFile, int offset, int n, int flags){
+  VStatFile *p = (VStatFile *)pFile;
+  return p->pReal->pMethods->xShmLock(p->pReal, offset, n, flags);
+}
+
+/* Memory barrier operation on shared memory */
+static void vstatShmBarrier(sqlite3_file *pFile){
+  VStatFile *p = (VStatFile *)pFile;
+  p->pReal->pMethods->xShmBarrier(p->pReal);
+}
+
+/* Unmap a shared memory segment */
+static int vstatShmUnmap(sqlite3_file *pFile, int deleteFlag){
+  VStatFile *p = (VStatFile *)pFile;
+  return p->pReal->pMethods->xShmUnmap(p->pReal, deleteFlag);
+}
+
+/* Fetch a page of a memory-mapped file */
+static int vstatFetch(
+  sqlite3_file *pFile,
+  sqlite3_int64 iOfst,
+  int iAmt,
+  void **pp
+){
+  VStatFile *p = (VStatFile *)pFile;
+  return p->pReal->pMethods->xFetch(p->pReal, iOfst, iAmt, pp);
+}
+
+/* Release a memory-mapped page */
+static int vstatUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
+  VStatFile *p = (VStatFile *)pFile;
+  return p->pReal->pMethods->xUnfetch(p->pReal, iOfst, pPage);
+}
+
+/*
+** Open an vstat file handle.
+*/
+static int vstatOpen(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_file *pFile,
+  int flags,
+  int *pOutFlags
+){
+  int rc;
+  VStatFile *p = (VStatFile*)pFile;
+
+  p->pReal = (sqlite3_file*)&p[1];
+  rc = REALVFS(pVfs)->xOpen(REALVFS(pVfs), zName, p->pReal, flags, pOutFlags);
+  if( flags & SQLITE_OPEN_MAIN_DB ){
+    p->eFiletype = VFSSTAT_MAIN;
+  }else if( flags & SQLITE_OPEN_MAIN_JOURNAL ){
+    p->eFiletype = VFSSTAT_JOURNAL;
+  }else if( flags & SQLITE_OPEN_WAL ){
+    p->eFiletype = VFSSTAT_WAL;
+  }else if( flags & SQLITE_OPEN_MASTER_JOURNAL ){
+    p->eFiletype = VFSSTAT_MASTERJRNL;
+  }else if( flags & SQLITE_OPEN_SUBJOURNAL ){
+    p->eFiletype = VFSSTAT_SUBJRNL;
+  }else if( flags & SQLITE_OPEN_TEMP_DB ){
+    p->eFiletype = VFSSTAT_TEMPDB;
+  }else if( flags & SQLITE_OPEN_TEMP_JOURNAL ){
+    p->eFiletype = VFSSTAT_TEMPJRNL;
+  }else{
+    p->eFiletype = VFSSTAT_TRANSIENT;
+  }
+  STATCNT(p->eFiletype,VFSSTAT_OPEN)++;
+  pFile->pMethods = rc ? 0 : &vstat_io_methods;
+  return rc;
+}
+
+/*
+** Delete the file located at zPath. If the dirSync argument is true,
+** ensure the file-system modifications are synced to disk before
+** returning.
+*/
+static int vstatDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
+  int rc;
+  rc = REALVFS(pVfs)->xDelete(REALVFS(pVfs), zPath, dirSync);
+  STATCNT(VFSSTAT_ANY,VFSSTAT_DELETE)++;
+  return rc;
+}
+
+/*
+** Test for access permissions. Return true if the requested permission
+** is available, or false otherwise.
+*/
+static int vstatAccess(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int flags, 
+  int *pResOut
+){
+  int rc;
+  rc = REALVFS(pVfs)->xAccess(REALVFS(pVfs), zPath, flags, pResOut);
+  STATCNT(VFSSTAT_ANY,VFSSTAT_ACCESS)++;
+  return rc;
+}
+
+/*
+** Populate buffer zOut with the full canonical pathname corresponding
+** to the pathname in zPath. zOut is guaranteed to point to a buffer
+** of at least (INST_MAX_PATHNAME+1) bytes.
+*/
+static int vstatFullPathname(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int nOut, 
+  char *zOut
+){
+  STATCNT(VFSSTAT_ANY,VFSSTAT_FULLPATH)++;
+  return REALVFS(pVfs)->xFullPathname(REALVFS(pVfs), zPath, nOut, zOut);
+}
+
+/*
+** Open the dynamic library located at zPath and return a handle.
+*/
+static void *vstatDlOpen(sqlite3_vfs *pVfs, const char *zPath){
+  return REALVFS(pVfs)->xDlOpen(REALVFS(pVfs), zPath);
+}
+
+/*
+** Populate the buffer zErrMsg (size nByte bytes) with a human readable
+** utf-8 string describing the most recent error encountered associated 
+** with dynamic libraries.
+*/
+static void vstatDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
+  REALVFS(pVfs)->xDlError(REALVFS(pVfs), nByte, zErrMsg);
+}
+
+/*
+** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
+*/
+static void (*vstatDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
+  return REALVFS(pVfs)->xDlSym(REALVFS(pVfs), p, zSym);
+}
+
+/*
+** Close the dynamic library handle pHandle.
+*/
+static void vstatDlClose(sqlite3_vfs *pVfs, void *pHandle){
+  REALVFS(pVfs)->xDlClose(REALVFS(pVfs), pHandle);
+}
+
+/*
+** Populate the buffer pointed to by zBufOut with nByte bytes of 
+** random data.
+*/
+static int vstatRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+  STATCNT(VFSSTAT_ANY,VFSSTAT_RANDOM)++;
+  return REALVFS(pVfs)->xRandomness(REALVFS(pVfs), nByte, zBufOut);
+}
+
+/*
+** Sleep for nMicro microseconds. Return the number of microseconds 
+** actually slept.
+*/
+static int vstatSleep(sqlite3_vfs *pVfs, int nMicro){
+  STATCNT(VFSSTAT_ANY,VFSSTAT_SLEEP)++;
+  return REALVFS(pVfs)->xSleep(REALVFS(pVfs), nMicro);
+}
+
+/*
+** Return the current time as a Julian Day number in *pTimeOut.
+*/
+static int vstatCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
+  STATCNT(VFSSTAT_ANY,VFSSTAT_CURTIME)++;
+  return REALVFS(pVfs)->xCurrentTime(REALVFS(pVfs), pTimeOut);
+}
+
+static int vstatGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+  return REALVFS(pVfs)->xGetLastError(REALVFS(pVfs), a, b);
+}
+static int vstatCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
+  STATCNT(VFSSTAT_ANY,VFSSTAT_CURTIME)++;
+  return REALVFS(pVfs)->xCurrentTimeInt64(REALVFS(pVfs), p);
+}
+
+/*
+** A virtual table for accessing the stats collected by this VFS shim
+*/
+static int vstattabConnect(sqlite3*, void*, int, const char*const*, 
+                           sqlite3_vtab**,char**);
+static int vstattabBestIndex(sqlite3_vtab*,sqlite3_index_info*);
+static int vstattabDisconnect(sqlite3_vtab*);
+static int vstattabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**);
+static int vstattabClose(sqlite3_vtab_cursor*);
+static int vstattabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr,
+                          int argc, sqlite3_value **argv);
+static int vstattabNext(sqlite3_vtab_cursor*);
+static int vstattabEof(sqlite3_vtab_cursor*);
+static int vstattabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int);
+static int vstattabRowid(sqlite3_vtab_cursor*,sqlite3_int64*);
+static int vstattabUpdate(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
+
+/* A cursor for the vfsstat virtual table */
+typedef struct VfsStatCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  int i;                          /* Pointing to this aVfsCnt[] value */
+} VfsStatCursor;
+
+
+static int vstattabConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  sqlite3_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define VSTAT_COLUMN_FILE  0 
+#define VSTAT_COLUMN_STAT  1
+#define VSTAT_COLUMN_COUNT 2
+
+  rc = sqlite3_declare_vtab(db,"CREATE TABLE x(file,stat,count)");
+  if( rc==SQLITE_OK ){
+    pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) );
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for vstat table object.
+*/
+static int vstattabDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new vstat table cursor object.
+*/
+static int vstattabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  VfsStatCursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+
+/*
+** Destructor for a VfsStatCursor.
+*/
+static int vstattabClose(sqlite3_vtab_cursor *cur){
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a VfsStatCursor to its next row of output.
+*/
+static int vstattabNext(sqlite3_vtab_cursor *cur){
+  ((VfsStatCursor*)cur)->i++;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the VfsStatCursor
+** is currently pointing.
+*/
+static int vstattabColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  VfsStatCursor *pCur = (VfsStatCursor*)cur;
+  switch( i ){
+    case VSTAT_COLUMN_FILE: {
+      sqlite3_result_text(ctx, azFile[pCur->i/VFSSTAT_nStat], -1, SQLITE_STATIC);
+      break;
+    }
+    case VSTAT_COLUMN_STAT: {
+      const char **az;
+      az = (pCur->i/VFSSTAT_nStat)==VFSSTAT_ANY ? azStatAny : azStat;
+      sqlite3_result_text(ctx, az[pCur->i%VFSSTAT_nStat], -1, SQLITE_STATIC);
+      break;
+    }
+    case VSTAT_COLUMN_COUNT: {
+      sqlite3_result_int64(ctx, aVfsCnt[pCur->i]);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.
+*/
+static int vstattabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  VfsStatCursor *pCur = (VfsStatCursor*)cur;
+  *pRowid = pCur->i;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int vstattabEof(sqlite3_vtab_cursor *cur){
+  VfsStatCursor *pCur = (VfsStatCursor*)cur;
+  return pCur->i >= VFSSTAT_MXCNT;
+}
+
+/*
+** Only a full table scan is supported.  So xFilter simply rewinds to
+** the beginning.
+*/
+static int vstattabFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  VfsStatCursor *pCur = (VfsStatCursor*)pVtabCursor;
+  pCur->i = 0;
+  return SQLITE_OK;
+}
+
+/*
+** Only a forwards full table scan is supported.  xBestIndex is a no-op.
+*/
+static int vstattabBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  return SQLITE_OK;
+}
+
+/*
+** Any VSTAT_COLUMN_COUNT can be changed to a positive integer.
+** No deletions or insertions are allowed.  No changes to other
+** columns are allowed.
+*/
+static int vstattabUpdate(
+  sqlite3_vtab *tab,
+  int argc, sqlite3_value **argv,
+  sqlite3_int64 *pRowid
+){
+  sqlite3_int64 iRowid, x;
+  if( argc==1 ) return SQLITE_ERROR;
+  if( sqlite3_value_type(argv[0])!=SQLITE_INTEGER ) return SQLITE_ERROR;
+  iRowid = sqlite3_value_int64(argv[0]);
+  if( iRowid!=sqlite3_value_int64(argv[1]) ) return SQLITE_ERROR;
+  if( iRowid<0 || iRowid>=VFSSTAT_MXCNT ) return SQLITE_ERROR;
+  if( sqlite3_value_type(argv[VSTAT_COLUMN_COUNT+2])!=SQLITE_INTEGER ){
+    return SQLITE_ERROR;
+  }
+  x = sqlite3_value_int64(argv[VSTAT_COLUMN_COUNT+2]);
+  if( x<0 ) return SQLITE_ERROR;
+  aVfsCnt[iRowid] = x;
+  return SQLITE_OK;
+}
+
+static sqlite3_module VfsStatModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  vstattabConnect,           /* xConnect */
+  vstattabBestIndex,         /* xBestIndex */
+  vstattabDisconnect,        /* xDisconnect */
+  0,                         /* xDestroy */
+  vstattabOpen,              /* xOpen - open a cursor */
+  vstattabClose,             /* xClose - close a cursor */
+  vstattabFilter,            /* xFilter - configure scan constraints */
+  vstattabNext,              /* xNext - advance a cursor */
+  vstattabEof,               /* xEof - check for end of scan */
+  vstattabColumn,            /* xColumn - read data */
+  vstattabRowid,             /* xRowid - read data */
+  vstattabUpdate,            /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+};
+
+/*
+** This routine is an sqlite3_auto_extension() callback, invoked to register
+** the vfsstat virtual table for all new database connections.
+*/
+static int vstatRegister(
+  sqlite3 *db,
+  const char **pzErrMsg,
+  const struct sqlite3_api_routines *pThunk
+){
+  return sqlite3_create_module(db, "vfsstat", &VfsStatModule, 0);
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+/* 
+** This routine is called when the extension is loaded.
+**
+** Register the new VFS.  Make arrangement to register the virtual table
+** for each new database connection.
+*/
+int sqlite3_vfsstat_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  vstat_vfs.pVfs = sqlite3_vfs_find(0);
+  vstat_vfs.base.szOsFile = sizeof(VStatFile) + vstat_vfs.pVfs->szOsFile;
+  rc = sqlite3_vfs_register(&vstat_vfs.base, 1);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_auto_extension((void(*)(void))vstatRegister);
+  }
+  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
+  return rc;
+}

ext/rbu/rbu1.test

@@ -10,10 +10,7 @@
 #***********************************************************************
 #
 
-if {![info exists testdir]} {
-  set testdir [file join [file dirname [info script]] .. .. test]
-}
-source $testdir/tester.tcl
+source [file join [file dirname [info script]] rbu_common.tcl]
 set ::testprefix rbu1
 
 db close
@@ -96,26 +93,6 @@ proc create_rbu5 {filename} {
   return $filename
 }
 
-# Run the RBU in file $rbu on target database $target until completion.
-#
-proc run_rbu {target rbu} {
-  sqlite3rbu rbu $target $rbu
-  while 1 {
-    set rc [rbu step]
-    if {$rc!="SQLITE_OK"} break
-  }
-  rbu close
-}
-
-proc step_rbu {target rbu} {
-  while 1 {
-    sqlite3rbu rbu $target $rbu
-    set rc [rbu step]
-    rbu close
-    if {$rc != "SQLITE_OK"} break
-  }
-  set rc
-}
 
 # Same as [step_rbu], except using a URI to open the target db.
 #
@@ -641,10 +618,25 @@ foreach {tn3 create_vfs destroy_vfs} {
   # correctly.
   reset_db
   forcedelete rbu.db
-  do_test $tn3.8 {
+  do_test $tn3.8.1 {
     list [catch { run_rbu test.db rbu.db } msg] $msg
   } {0 SQLITE_DONE}
-  
+
+  # Test that an RBU database containing only empty data_xxx tables is
+  # also handled correctly.
+  reset_db
+  forcedelete rbu.db
+  do_execsql_test $tn3.8.2.1 {
+    CREATE TABLE t1(a PRIMARY KEY, b);
+    INSERT INTO t1 VALUES(1, 2);
+    ATTACH 'rbu.db' AS rbu;
+    CREATE TABLE data_t1(a, b, rbu_control);
+    DETACH rbu;
+  }
+  do_test $tn3.8.2.1 {
+    list [catch { run_rbu test.db rbu.db } msg] $msg
+  } {0 SQLITE_DONE}
+
   # Test that RBU can update indexes containing NULL values.
   #
   reset_db

ext/rbu/rbu5.test

@@ -12,35 +12,10 @@
 # Test some properties of the pager_rbu_mode and rbu_mode pragmas.
 #
 
-if {![info exists testdir]} {
-  set testdir [file join [file dirname [info script]] .. .. test]
-}
-source $testdir/tester.tcl
+source [file join [file dirname [info script]] rbu_common.tcl]
 set ::testprefix rbu5
 
 
-# Run the RBU in file $rbu on target database $target until completion.
-#
-proc run_rbu {target rbu} {
-  sqlite3rbu rbu $target $rbu
-  while { [rbu step]=="SQLITE_OK" } {}
-  rbu close
-}
-
-
-# Run the RBU in file $rbu on target database $target one step at a
-# time until completion.
-#
-proc step_rbu {target rbu} {
-  while 1 {
-    sqlite3rbu rbu $target $rbu
-    set rc [rbu step]
-    rbu close
-    if {$rc != "SQLITE_OK"} break
-  }
-  set rc
-}
-
 # Return a list of the primary key columns for table $tbl in the database
 # opened by database handle $db.
 #

ext/rbu/rbu_common.tcl

@@ -15,12 +15,43 @@ if {![info exists testdir]} {
 }
 source $testdir/tester.tcl
 
+proc check_prestep_state {target state} {
+  set oal_exists [file exists $target-oal]
+  set wal_exists [file exists $target-wal]
+  set progress [rbu progress]
+
+  if {($progress==0 && $state!="oal" && $state!="done")
+   || ($oal_exists && $wal_exists)
+   || ($progress>0 && $state=="oal" && (!$oal_exists || $wal_exists))
+   || ($state=="move" && (!$oal_exists || $wal_exists))
+   || ($state=="checkpoint" && ($oal_exists || !$wal_exists))
+   || ($state=="done" && ($oal_exists && $progress!=0))
+  } {
+    error "B: state=$state progress=$progress oal=$oal_exists wal=$wal_exists"
+  }
+}
+
+proc check_poststep_state {rc target state} {
+  if {$rc=="SQLITE_OK" || $rc=="SQLITE_DONE"} {
+    set oal_exists [file exists $target-oal]
+    set wal_exists [file exists $target-wal]
+    if {$state=="move" && ($oal_exists || !$wal_exists)} {
+      error "A: state=$state progress=$progress oal=$oal_exists wal=$wal_exists"
+    }
+  }
+}
+
 # Run the RBU in file $rbu on target database $target until completion.
 #
 proc run_rbu {target rbu} {
   sqlite3rbu rbu $target $rbu
   while 1 {
+    set state [rbu state]
+
+    check_prestep_state $target $state
     set rc [rbu step]
+    check_poststep_state $rc $target $state
+
     if {$rc!="SQLITE_OK"} break
   }
   rbu close
@@ -29,10 +60,33 @@ proc run_rbu {target rbu} {
 proc step_rbu {target rbu} {
   while 1 {
     sqlite3rbu rbu $target $rbu
+    set state [rbu state]
+    check_prestep_state $target $state
     set rc [rbu step]
+    check_poststep_state $rc $target $state
     rbu close
     if {$rc != "SQLITE_OK"} break
   }
   set rc
 }
 
+proc do_rbu_vacuum_test {tn step} {
+  uplevel [list do_test $tn.1 {
+    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
+    while 1 {
+      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
+      set state [rbu state]
+      check_prestep_state test.db $state
+      set rc [rbu step]
+      check_poststep_state $rc test.db $state
+      if {$rc!="SQLITE_OK"} break
+      if {$step==1} { rbu close }
+    }
+    rbu close
+  } {SQLITE_DONE}]
+
+  uplevel [list do_execsql_test $tn.2 {
+    PRAGMA integrity_check
+  } ok]
+}
+

ext/rbu/rbudiff.test

@@ -25,6 +25,10 @@ proc get_rbudiff_sql {db1 db2} {
   exec $::PROG --rbu $db1 $db2
 }
 
+proc get_vtab_rbudiff_sql {db1 db2} {
+  exec $::PROG --vtab --rbu $db1 $db2
+}
+
 proc step_rbu {target rbu} {
   while 1 {
     sqlite3rbu rbu $target $rbu
@@ -44,6 +48,11 @@ proc apply_rbudiff {sql target} {
   step_rbu $target rbu.db
 }
 
+proc sqlesc {id} {
+  set ret "'[string map {' ''} $id]'"
+  set ret
+}
+
 # The only argument is the output of an [sqldiff -rbu] run. This command
 # tests that the contents of the rbu_count table is correct. An exception
 # is thrown if it is not.
@@ -54,7 +63,7 @@ proc test_rbucount {sql} {
   tmpdb eval {
     SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table'
   } {
-    set a [tmpdb eval "SELECT count(*) FROM $name"]
+    set a [tmpdb eval "SELECT count(*) FROM [sqlesc $name]"]
     set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}]
     if {$a != $b} { 
       tmpdb close
@@ -71,9 +80,11 @@ proc rbudiff_cksum {db1} {
   sqlite3 dbtmp $db1
   foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] {
     set cols [list]
-    dbtmp eval "PRAGMA table_info = $tbl" { lappend cols "quote( $name )" }
+    dbtmp eval "PRAGMA table_info = [sqlesc $tbl]" { 
+      lappend cols "quote( $name )" 
+    }
     append txt [dbtmp eval \
-      "SELECT [join $cols {||'.'||}] FROM $tbl ORDER BY 1"
+      "SELECT [join $cols {||'.'||}] FROM [sqlesc $tbl] ORDER BY 1"
     ]
   }
   dbtmp close
@@ -159,5 +170,57 @@ foreach {tn init mod} {
   do_test 1.$tn.5 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
 }
 
+#-------------------------------------------------------------------------
+# Test that if the --vtab switch is present, [sqldiff] handles virtual
+# table types fts[345] and rtree correctly.
+#
+ifcapable fts3&&fts5&&rtree {
+
+foreach {tn init mod} {
+  1 {
+    CREATE VIRTUAL TABLE t1 USING fts5(c);
+    INSERT INTO t1 VALUES('a b c');
+    INSERT INTO t1 VALUES('a b c');
+  } {
+    DELETE FROM t1 WHERE rowid = 1;
+    INSERT INTO t1 VALUES('a b c');
+  }
+
+  2 {
+    CREATE VIRTUAL TABLE "x y" USING 'rtree'(id, x1, x2);
+    INSERT INTO "x y" VALUES(1, 2, 3);
+    INSERT INTO "x y" VALUES(2, 4, 6);
+  } {
+    DELETE FROM "x y" WHERE rowid = 1;
+    INSERT INTO "x y" VALUES(3, 6, 9);
+  }
+
+  3 {
+    CREATE VIRTUAL TABLE 'x''y' USING fts3;
+    INSERT INTO 'x''y' VALUES('one two three');
+    INSERT INTO 'x''y' VALUES('four five six');
+  } {
+    DELETE FROM 'x''y' WHERE rowid = 1;
+    INSERT INTO 'x''y' VALUES('one two three');
+  }
+
+} {
+
+  forcedelete test.db test.db2
+  sqlite3 db test.db
+  db eval "$init"
+  sqlite3 db test.db2
+  db eval "$init ; $mod"
+  db close
+
+  do_test 2.$tn.1 {
+    set sql [get_vtab_rbudiff_sql test.db test.db2]
+    apply_rbudiff $sql test.db
+  } {SQLITE_DONE}
+  do_test 2.$tn.2 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
+}
+
+}
+
 finish_test
 

ext/rbu/rbuvacuum.test

@@ -17,23 +17,6 @@
 source [file join [file dirname [info script]] rbu_common.tcl]
 set ::testprefix rbuvacuum
 
-proc do_rbu_vacuum_test {tn step} {
-  uplevel [list do_test $tn.1 {
-    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
-    while 1 {
-      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
-      set rc [rbu step]
-      if {$rc!="SQLITE_OK"} break
-      if {$step==1} { rbu close }
-    }
-    rbu close
-  } {SQLITE_DONE}]
-
-  uplevel [list do_execsql_test $tn.2 {
-    PRAGMA integrity_check
-  } ok]
-}
-
 foreach step {0 1} {
 
   set ::testprefix rbuvacuum-step=$step
@@ -404,7 +387,6 @@ do_test 3.5 {
   list [catch { rbu close } msg] $msg
 } {0 SQLITE_DONE}
 
-
 catch { db close }
 finish_test
 

ext/rbu/rbuvacuum2.test

@@ -0,0 +1,162 @@
+# 2016 June 1
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the RBU module. More specifically, it
+# contains tests to ensure that the sqlite3rbu_vacuum() API works as
+# expected.
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+
+foreach step {0 1} {
+  set ::testprefix rbuvacuum2-$step
+  
+  #-------------------------------------------------------------------------
+  # Test that a database that contains fts3 tables can be vacuumed.
+  #
+  ifcapable fts3 {
+    reset_db
+    do_execsql_test 1.1 {
+      CREATE VIRTUAL TABLE t1 USING fts3(z, y);
+      INSERT INTO t1 VALUES('fix this issue', 'at some point');
+    }
+  
+    do_rbu_vacuum_test 1.2 $step
+  
+    do_execsql_test 1.3 {
+      SELECT * FROM t1;
+    } {{fix this issue} {at some point}}
+  
+    do_execsql_test 1.4 {
+      SELECT rowid FROM t1 WHERE t1 MATCH 'fix';
+    } {1}
+
+    do_execsql_test 1.5 {
+      INSERT INTO t1 VALUES('a b c', 'd e f');
+      INSERT INTO t1 VALUES('l h i', 'd e f');
+      DELETE FROM t1 WHERE docid = 2;
+      INSERT INTO t1 VALUES('a b c', 'x y z');
+    }
+
+    do_rbu_vacuum_test 1.6 $step
+    do_execsql_test 1.7 {
+      INSERT INTO t1(t1) VALUES('integrity-check');
+      SELECT * FROM t1;
+    } {
+      {fix this issue} {at some point}
+      {l h i} {d e f}
+      {a b c} {x y z}
+    }
+  }
+  
+  #-------------------------------------------------------------------------
+  # Test that a database that contains fts5 tables can be vacuumed.
+  #
+  ifcapable fts5 {
+    reset_db
+    do_execsql_test 2.1 {
+      CREATE VIRTUAL TABLE t1 USING fts5(z, y);
+      INSERT INTO t1 VALUES('fix this issue', 'at some point');
+    }
+  
+    do_rbu_vacuum_test 2.2 $step
+  
+    do_execsql_test 2.3 {
+      SELECT * FROM t1;
+    } {{fix this issue} {at some point}}
+  
+    do_execsql_test 2.4 {
+      SELECT rowid FROM t1 ('fix');
+    } {1}
+
+    do_execsql_test 2.5 {
+      INSERT INTO t1 VALUES('a b c', 'd e f');
+      INSERT INTO t1 VALUES('l h i', 'd e f');
+      DELETE FROM t1 WHERE rowid = 2;
+      INSERT INTO t1 VALUES('a b c', 'x y z');
+    }
+
+    do_rbu_vacuum_test 2.6 $step
+    do_execsql_test 2.7 {
+      INSERT INTO t1(t1) VALUES('integrity-check');
+      SELECT * FROM t1;
+    } {
+      {fix this issue} {at some point}
+      {l h i} {d e f}
+      {a b c} {x y z}
+    }
+  }
+
+  #-------------------------------------------------------------------------
+  # Test that a database that contains an rtree table can be vacuumed.
+  #
+  ifcapable rtree {
+    reset_db
+    do_execsql_test 3.1 {
+      CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2);
+      INSERT INTO rt VALUES(1, 45, 55);
+      INSERT INTO rt VALUES(2, 50, 60);
+      INSERT INTO rt VALUES(3, 55, 65);
+    }
+  
+    do_rbu_vacuum_test 3.2 $step
+  
+    do_execsql_test 3.3 {
+      SELECT * FROM rt;
+    } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0}
+  
+    do_execsql_test 3.4.1 {
+      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
+    } {1 2}
+    do_execsql_test 3.4.2 {
+      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
+    } {2 3}
+
+    do_rbu_vacuum_test 3.5 $step
+
+    do_execsql_test 3.6.1 {
+      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
+    } {1 2}
+    do_execsql_test 3.6.2 {
+      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
+    } {2 3}
+  }
+
+  ifcapable trigger {
+    reset_db
+    do_execsql_test 4.1 {
+      CREATE TABLE t1(a, b, c);
+      INSERT INTO t1 VALUES(1, 2, 3);
+      CREATE VIEW v1 AS SELECT * FROM t1;
+      CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END;
+    }
+
+    do_execsql_test 4.2 {
+      SELECT * FROM sqlite_master;
+    } {
+    table t1 t1 2 {CREATE TABLE t1(a, b, c)}
+    view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
+    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
+    }
+
+    do_rbu_vacuum_test 4.3 $step
+    do_execsql_test 4.4 {
+      SELECT * FROM sqlite_master;
+    } {
+    table t1 t1 2 {CREATE TABLE t1(a, b, c)}
+    view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
+    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
+    }
+  }
+
+}
+
+finish_test

ext/rbu/sqlite3rbu.c

@@ -921,12 +921,14 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){
   int rc;
   memset(pIter, 0, sizeof(RbuObjIter));
 
-  rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, 
+  rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, 
+    sqlite3_mprintf(
       "SELECT rbu_target_name(name, type='view') AS target, name "
       "FROM sqlite_master "
       "WHERE type IN ('table', 'view') AND target IS NOT NULL "
+      " %s "
       "ORDER BY name"
-  );
+  , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : ""));
 
   if( rc==SQLITE_OK ){
     rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg,
@@ -2505,9 +2507,9 @@ static void rbuFileSuffix3(const char *zBase, char *z){
 #endif
   {
     int i, sz;
-    sz = sqlite3Strlen30(z);
+    sz = (int)strlen(z)&0xffffff;
     for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
-    if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4);
+    if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4);
   }
 #endif
 }
@@ -3553,30 +3555,7 @@ static sqlite3rbu *openRbuHandle(
     if( p->rc==SQLITE_OK ){
       if( p->eStage==RBU_STAGE_OAL ){
         sqlite3 *db = p->dbMain;
-
-        if( pState->eStage==0 && rbuIsVacuum(p) ){
-          rbuCopyPragma(p, "page_size");
-          rbuCopyPragma(p, "auto_vacuum");
-        }
-
-        /* Open transactions both databases. The *-oal file is opened or
-        ** created at this point. */
-        if( p->rc==SQLITE_OK ){
-          p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
-        }
-        if( p->rc==SQLITE_OK ){
-          p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg);
-        }
-
-        /* Check if the main database is a zipvfs db. If it is, set the upper
-        ** level pager to use "journal_mode=off". This prevents it from 
-        ** generating a large journal using a temp file.  */
-        if( p->rc==SQLITE_OK ){
-          int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0);
-          if( frc==SQLITE_OK ){
-            p->rc = sqlite3_exec(db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg);
-          }
-        }
+        p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg);
 
         /* Point the object iterator at the first object */
         if( p->rc==SQLITE_OK ){
@@ -3587,12 +3566,34 @@ static sqlite3rbu *openRbuHandle(
         ** update finished.  */
         if( p->rc==SQLITE_OK && p->objiter.zTbl==0 ){
           p->rc = SQLITE_DONE;
-        }
+          p->eStage = RBU_STAGE_DONE;
+        }else{
+          if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){
+            rbuCopyPragma(p, "page_size");
+            rbuCopyPragma(p, "auto_vacuum");
+          }
 
-        if( p->rc==SQLITE_OK ){
-          rbuSetupOal(p, pState);
-        }
+          /* Open transactions both databases. The *-oal file is opened or
+          ** created at this point. */
+          if( p->rc==SQLITE_OK ){
+            p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
+          }
 
+          /* Check if the main database is a zipvfs db. If it is, set the upper
+          ** level pager to use "journal_mode=off". This prevents it from 
+          ** generating a large journal using a temp file.  */
+          if( p->rc==SQLITE_OK ){
+            int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0);
+            if( frc==SQLITE_OK ){
+              p->rc = sqlite3_exec(
+                db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg);
+            }
+          }
+
+          if( p->rc==SQLITE_OK ){
+            rbuSetupOal(p, pState);
+          }
+        }
       }else if( p->eStage==RBU_STAGE_MOVE ){
         /* no-op */
       }else if( p->eStage==RBU_STAGE_CKPT ){
@@ -3759,9 +3760,39 @@ void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
   }
 }
 
+/*
+** Return the current state of the RBU vacuum or update operation.
+*/
+int sqlite3rbu_state(sqlite3rbu *p){
+  int aRes[] = {
+    0, SQLITE_RBU_STATE_OAL, SQLITE_RBU_STATE_MOVE,
+    0, SQLITE_RBU_STATE_CHECKPOINT, SQLITE_RBU_STATE_DONE
+  };
+
+  assert( RBU_STAGE_OAL==1 );
+  assert( RBU_STAGE_MOVE==2 );
+  assert( RBU_STAGE_CKPT==4 );
+  assert( RBU_STAGE_DONE==5 );
+  assert( aRes[RBU_STAGE_OAL]==SQLITE_RBU_STATE_OAL );
+  assert( aRes[RBU_STAGE_MOVE]==SQLITE_RBU_STATE_MOVE );
+  assert( aRes[RBU_STAGE_CKPT]==SQLITE_RBU_STATE_CHECKPOINT );
+  assert( aRes[RBU_STAGE_DONE]==SQLITE_RBU_STATE_DONE );
+
+  if( p->rc!=SQLITE_OK && p->rc!=SQLITE_DONE ){
+    return SQLITE_RBU_STATE_ERROR;
+  }else{
+    assert( p->rc!=SQLITE_DONE || p->eStage==RBU_STAGE_DONE );
+    assert( p->eStage==RBU_STAGE_OAL
+         || p->eStage==RBU_STAGE_MOVE
+         || p->eStage==RBU_STAGE_CKPT
+         || p->eStage==RBU_STAGE_DONE
+    );
+    return aRes[p->eStage];
+  }
+}
+
 int sqlite3rbu_savestate(sqlite3rbu *p){
   int rc = p->rc;
-  
   if( rc==SQLITE_DONE ) return SQLITE_OK;
 
   assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE );

ext/rbu/sqlite3rbu.h

@@ -474,6 +474,44 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu);
 */
 void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo);
 
+/*
+** Obtain an indication as to the current stage of an RBU update or vacuum.
+** This function always returns one of the SQLITE_RBU_STATE_XXX constants
+** defined in this file. Return values should be interpreted as follows:
+**
+** SQLITE_RBU_STATE_OAL:
+**   RBU is currently building a *-oal file. The next call to sqlite3rbu_step()
+**   may either add further data to the *-oal file, or compute data that will
+**   be added by a subsequent call.
+**
+** SQLITE_RBU_STATE_MOVE:
+**   RBU has finished building the *-oal file. The next call to sqlite3rbu_step()
+**   will move the *-oal file to the equivalent *-wal path. If the current
+**   operation is an RBU update, then the updated version of the database
+**   file will become visible to ordinary SQLite clients following the next
+**   call to sqlite3rbu_step().
+**
+** SQLITE_RBU_STATE_CHECKPOINT:
+**   RBU is currently performing an incremental checkpoint. The next call to
+**   sqlite3rbu_step() will copy a page of data from the *-wal file into
+**   the target database file.
+**
+** SQLITE_RBU_STATE_DONE:
+**   The RBU operation has finished. Any subsequent calls to sqlite3rbu_step()
+**   will immediately return SQLITE_DONE.
+**
+** SQLITE_RBU_STATE_ERROR:
+**   An error has occurred. Any subsequent calls to sqlite3rbu_step() will
+**   immediately return the SQLite error code associated with the error.
+*/
+#define SQLITE_RBU_STATE_OAL        1
+#define SQLITE_RBU_STATE_MOVE       2
+#define SQLITE_RBU_STATE_CHECKPOINT 3
+#define SQLITE_RBU_STATE_DONE       4
+#define SQLITE_RBU_STATE_ERROR      5
+
+int sqlite3rbu_state(sqlite3rbu *pRbu);
+
 /*
 ** Create an RBU VFS named zName that accesses the underlying file-system
 ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, 

ext/rbu/test_rbu.c

@@ -69,6 +69,8 @@ static int test_sqlite3rbu_cmd(
     {"dbMain_eval", 3, "SQL"},    /* 4 */
     {"bp_progress", 2, ""},       /* 5 */
     {"db", 3, "RBU"},             /* 6 */
+    {"state", 2, ""},             /* 7 */
+    {"progress", 2, ""},          /* 8 */
     {0,0,0}
   };
   int iCmd;
@@ -166,6 +168,18 @@ static int test_sqlite3rbu_cmd(
       }
       break;
     }
+    case 7: /* state */ {
+      const char *aRes[] = { 0, "oal", "move", "checkpoint", "done", "error" };
+      int eState = sqlite3rbu_state(pRbu);
+      assert( eState>0 && eState<=5 );
+      Tcl_SetResult(interp, (char*)aRes[eState], TCL_STATIC);
+      break;
+    }
+    case 8: /* progress */ {
+      sqlite3_int64 nStep =  sqlite3rbu_progress(pRbu);
+      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nStep));
+      break;
+    }
 
     default: /* seems unlikely */
       assert( !"cannot happen" );

ext/rtree/rtree.c

@@ -2800,6 +2800,53 @@ static RtreeValue rtreeValueUp(sqlite3_value *v){
 }
 #endif /* !defined(SQLITE_RTREE_INT_ONLY) */
 
+/*
+** A constraint has failed while inserting a row into an rtree table. 
+** Assuming no OOM error occurs, this function sets the error message 
+** (at pRtree->base.zErrMsg) to an appropriate value and returns
+** SQLITE_CONSTRAINT.
+**
+** Parameter iCol is the index of the leftmost column involved in the
+** constraint failure. If it is 0, then the constraint that failed is
+** the unique constraint on the id column. Otherwise, it is the rtree
+** (c1<=c2) constraint on columns iCol and iCol+1 that has failed.
+**
+** If an OOM occurs, SQLITE_NOMEM is returned instead of SQLITE_CONSTRAINT.
+*/
+static int rtreeConstraintError(Rtree *pRtree, int iCol){
+  sqlite3_stmt *pStmt = 0;
+  char *zSql; 
+  int rc;
+
+  assert( iCol==0 || iCol%2 );
+  zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", pRtree->zDb, pRtree->zName);
+  if( zSql ){
+    rc = sqlite3_prepare_v2(pRtree->db, zSql, -1, &pStmt, 0);
+  }else{
+    rc = SQLITE_NOMEM;
+  }
+  sqlite3_free(zSql);
+
+  if( rc==SQLITE_OK ){
+    if( iCol==0 ){
+      const char *zCol = sqlite3_column_name(pStmt, 0);
+      pRtree->base.zErrMsg = sqlite3_mprintf(
+          "UNIQUE constraint failed: %s.%s", pRtree->zName, zCol
+      );
+    }else{
+      const char *zCol1 = sqlite3_column_name(pStmt, iCol);
+      const char *zCol2 = sqlite3_column_name(pStmt, iCol+1);
+      pRtree->base.zErrMsg = sqlite3_mprintf(
+          "rtree constraint failed: %s.(%s<=%s)", pRtree->zName, zCol1, zCol2
+      );
+    }
+  }
+
+  sqlite3_finalize(pStmt);
+  return (rc==SQLITE_OK ? SQLITE_CONSTRAINT : rc);
+}
+
+
 
 /*
 ** The xUpdate method for rtree module virtual tables.
@@ -2850,7 +2897,7 @@ static int rtreeUpdate(
         cell.aCoord[ii].f = rtreeValueDown(azData[ii+3]);
         cell.aCoord[ii+1].f = rtreeValueUp(azData[ii+4]);
         if( cell.aCoord[ii].f>cell.aCoord[ii+1].f ){
-          rc = SQLITE_CONSTRAINT;
+          rc = rtreeConstraintError(pRtree, ii+1);
           goto constraint;
         }
       }
@@ -2861,7 +2908,7 @@ static int rtreeUpdate(
         cell.aCoord[ii].i = sqlite3_value_int(azData[ii+3]);
         cell.aCoord[ii+1].i = sqlite3_value_int(azData[ii+4]);
         if( cell.aCoord[ii].i>cell.aCoord[ii+1].i ){
-          rc = SQLITE_CONSTRAINT;
+          rc = rtreeConstraintError(pRtree, ii+1);
           goto constraint;
         }
       }
@@ -2882,7 +2929,7 @@ static int rtreeUpdate(
           if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){
             rc = rtreeDeleteRowid(pRtree, cell.iRowid);
           }else{
-            rc = SQLITE_CONSTRAINT;
+            rc = rtreeConstraintError(pRtree, 0);
             goto constraint;
           }
         }
@@ -2965,6 +3012,11 @@ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){
   int rc;
   i64 nRow = 0;
 
+  if( sqlite3_table_column_metadata(db,pRtree->zDb,"sqlite_stat1",
+          0,0,0,0,0,0)==SQLITE_ERROR ){
+    pRtree->nRowEst = RTREE_DEFAULT_ROWEST;
+    return SQLITE_OK;
+  }
   zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName);
   if( zSql==0 ){
     rc = SQLITE_NOMEM;

ext/rtree/rtree1.test

@@ -194,13 +194,13 @@ do_test rtree-2.1.3 {
 
 do_test rtree-2.2.1 {
   catchsql { INSERT INTO t1 VALUES(2, 1, 3, 2, 4) }
-} {1 {constraint failed}}
+} {1 {UNIQUE constraint failed: t1.ii}}
 do_test rtree-2.2.2 {
   catchsql { INSERT INTO t1 VALUES(4, 1, 3, 4, 2) }
-} {1 {constraint failed}}
+} {1 {rtree constraint failed: t1.(y1<=y2)}}
 do_test rtree-2.2.3 {
   catchsql { INSERT INTO t1 VALUES(4, 3, 1, 2, 4) }
-} {1 {constraint failed}}
+} {1 {rtree constraint failed: t1.(x1<=x2)}}
 do_test rtree-2.2.4 {
   execsql { SELECT ii FROM t1 ORDER BY ii }
 } {1 2 3}
@@ -236,7 +236,7 @@ do_test rtree-3.1.3 {
 # Test the constraint on the coordinates (c[i]<=c[i+1] where (i%2==0)):
 do_test rtree-3.2.1 {
   catchsql { INSERT INTO t1 VALUES(7, 2, 6, 4, 3) }
-} {1 {constraint failed}}
+} {1 {rtree constraint failed: t1.(y1<=y2)}}
 do_test rtree-3.2.2 {
   catchsql { INSERT INTO t1 VALUES(8, 2, 6, 3, 3) }
 } {0 {}}
@@ -490,11 +490,11 @@ foreach {tn sql_template testdata} {
   }
 
   4    "INSERT %CONF% INTO t1 VALUES(2, 7, 6, 7, 7)" {
-    ROLLBACK 0 1 {1 1 2 3 4   2 2 3 4 5   3 3 4 5 6}
-    ABORT    0 1 {1 1 2 3 4   2 2 3 4 5   3 3 4 5 6   4 4 5 6 7}
+    ROLLBACK 0 2 {1 1 2 3 4   2 2 3 4 5   3 3 4 5 6}
+    ABORT    0 2 {1 1 2 3 4   2 2 3 4 5   3 3 4 5 6   4 4 5 6 7}
     IGNORE   0 0 {1 1 2 3 4   2 2 3 4 5   3 3 4 5 6   4 4 5 6 7}
-    FAIL     0 1 {1 1 2 3 4   2 2 3 4 5   3 3 4 5 6   4 4 5 6 7}
-    REPLACE  0 1 {1 1 2 3 4   2 2 3 4 5   3 3 4 5 6   4 4 5 6 7}
+    FAIL     0 2 {1 1 2 3 4   2 2 3 4 5   3 3 4 5 6   4 4 5 6 7}
+    REPLACE  0 2 {1 1 2 3 4   2 2 3 4 5   3 3 4 5 6   4 4 5 6 7}
   }
 
 } {
@@ -510,7 +510,9 @@ foreach {tn sql_template testdata} {
     }
 
     set res(0) {0 {}}
-    set res(1) {1 {constraint failed}}
+    set res(1) {1 {UNIQUE constraint failed: t1.idx}}
+    set res(2) {1 {rtree constraint failed: t1.(x1<=x2)}}
+
     do_catchsql_test $testname.1 $sql $res($error)
     do_test $testname.2 [list sql_uses_stmt db $sql] $uses
     do_execsql_test $testname.3 { SELECT * FROM t1 ORDER BY idx } $data

ext/rtree/rtree3.test

@@ -47,7 +47,8 @@ ifcapable !rtree {
 #
 #   rtree3-8: Test OOM while registering the r-tree module with sqlite.
 #
-
+#   rtree3-11: OOM following a constraint failure
+#
 do_faultsim_test rtree3-1 -faults oom* -prep {
   faultsim_delete_and_reopen
 } -body {
@@ -234,4 +235,32 @@ do_faultsim_test rtree3-10 -faults oom-* -prep {
   faultsim_test_result {0 2}
 }
 
+
+do_test rtree3-11.prep {
+  faultsim_delete_and_reopen
+  execsql { 
+    CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
+    INSERT INTO rt VALUES(1, 2, 3, 4, 5);
+  }
+  faultsim_save_and_close
+} {}
+do_faultsim_test rtree3-10.1 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT * FROM rt }
+} -body {
+  execsql { INSERT INTO rt VALUES(1, 2, 3, 4, 5) }
+} -test {
+  faultsim_test_result {1 {UNIQUE constraint failed: rt.ii}} \
+                       {1 {constraint failed}}
+}
+do_faultsim_test rtree3-10.2 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT * FROM rt }
+} -body {
+  execsql { INSERT INTO rt VALUES(2, 2, 3, 5, 4) }
+} -test {
+  faultsim_test_result {1 {rtree constraint failed: rt.(y1<=y2)}} \
+                       {1 {constraint failed}}
+}
+
 finish_test

ext/rtree/rtreeC.test

@@ -350,7 +350,3 @@ do_eqp_execsql_test 7.4 {
 }
 
 finish_test
-
-
-
-finish_test

ext/rtree/rtreeG.test

@@ -0,0 +1,66 @@
+# 2016-05-32
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file contains tests for the r-tree module.
+#
+# Verify that no invalid SQL is run during initialization
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+ifcapable !rtree { finish_test ; return }
+
+db close
+sqlite3_shutdown
+test_sqlite3_log [list lappend ::log]
+set ::log [list]
+sqlite3 db test.db
+
+
+set ::log {}
+do_execsql_test rtreeG-1.1 {
+  CREATE VIRTUAL TABLE t1 USING rtree(id,x0,x1,y0,y1);
+} {}
+do_test rtreeG-1.1log {
+  set ::log
+} {}
+
+do_execsql_test rtreeG-1.2 {
+  INSERT INTO t1 VALUES(1,10,15,5,23),(2,20,21,5,23),(3,10,15,20,30);
+  SELECT id from t1 WHERE x0>8 AND x1<16 AND y0>2 AND y1<25;
+} {1}
+do_test rtreeG-1.2log {
+  set ::log
+} {}
+
+db close
+sqlite3 db test.db
+do_execsql_test rtreeG-1.3 {
+  SELECT id from t1 WHERE x0>8 AND x1<16 AND y0>2 AND y1<25;
+} {1}
+do_test rtreeG-1.3log {
+  set ::log
+} {}
+
+do_execsql_test rtreeG-1.4 {
+  DROP TABLE t1;
+} {}
+do_test rtreeG-1.4log {
+  set ::log
+} {}
+
+db close
+sqlite3_shutdown
+test_sqlite3_log
+sqlite3_initialize
+sqlite3 db test.db
+
+finish_test

main.mk

@@ -325,6 +325,7 @@ TESTSRC = \
 TESTSRC += \
   $(TOP)/ext/misc/amatch.c \
   $(TOP)/ext/misc/closure.c \
+  $(TOP)/ext/misc/csv.c \
   $(TOP)/ext/misc/eval.c \
   $(TOP)/ext/misc/fileio.c \
   $(TOP)/ext/misc/fuzzer.c \
@@ -450,7 +451,8 @@ TESTPROGS = \
   testfixture$(EXE) \
   sqlite3$(EXE) \
   sqlite3_analyzer$(EXE) \
-  sqldiff$(EXE)
+  sqldiff$(EXE) \
+  dbhash$(EXE)
 
 # Databases containing fuzzer test cases
 #
@@ -487,6 +489,13 @@ sqldiff$(EXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
 	$(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \
 		$(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB)
 
+dbhash$(EXE):	$(TOP)/tool/dbhash.c sqlite3.c sqlite3.h
+	$(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \
+		$(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB)
+
+scrub$(EXE):	$(TOP)/ext/misc/scrub.c sqlite3.o
+	$(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB)
+
 srcck1$(EXE):	$(TOP)/tool/srcck1.c
 	$(BCC) -o srcck1$(EXE) $(TOP)/tool/srcck1.c
 

manifest

@@ -1,10 +1,10 @@
-C Omit\sthe\sunnecessary\sWHERE_REOPEN_IDX\sflag\sin\sthe\sWHERE\sinterface.
-D 2016-05-09T19:58:56.729
-F Makefile.in 9eda6e1c90d05c199c3ec8a7069b0682ad307657
+C Add\sthe\ssqlite3rbu_state()\sAPI.\sUsed\sto\sdetermine\sthe\scurrent\sstate\s(creating\sOAL,\sready\sto\smove\sOAL,\sincremental-checkpoint,\sfinished\sor\serror)\sof\san\sRBU\soperation.
+D 2016-07-01T12:39:58.096
+F Makefile.in bc2b4864a23a4a21c3e26d7b4350f51bab324d45
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc db82b35aef27f412fef14d8534afc022138bcdfd
+F Makefile.msc 50149765ef72f4e652b9a0f1f6462c4784bb9423
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
-F VERSION 5d234da9b5dae329fab75ff75884cfe0a9cb3fda
+F VERSION cb29eb11e493dd85b3eeec4053c03949bf98478e
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
@@ -14,7 +14,7 @@ F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4
 F autoconf/Makefile.msc 0eca137c12542bd76c253a2d24380f29ade59b95
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
-F autoconf/configure.ac ad7001cdde3d6ae6053bc0b66009869d7c5012a8
+F autoconf/configure.ac cacf2616abf6e4a569bde2ef365c143caeec40bc
 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd
 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873
 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43
@@ -30,8 +30,8 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63
 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977
 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc
 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
-F configure df9f793ad47eaeb3648a9090fbc0b96955e88b5a x
-F configure.ac 02b972db825685b4b0cca0234b2aa97f71bd15a5
+F configure 25cf7eb52a88846e4c1978016a9db22763c54f41 x
+F configure.ac 480e1a17d5316b1751c8edcb15b8e6b8ab0a6cc3
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/lemon.html e2118945e5f07ed146b45c9cd2b2dd6eabb8ebf2
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
@@ -98,14 +98,14 @@ F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h b770c5e0a8d2ee071ddffc7ab722dbf3474a8abe
-F ext/fts5/fts5Int.h 3677076aecbf645a7f2a019115c6a4ec3272dd78
+F ext/fts5/fts5Int.h 9bd0c7c64285b5b368eca0ac63613185c5ad24ba
 F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
-F ext/fts5/fts5_expr.c da2b33c2aac91e96641c0a7cf2bbaa36eb2667f7
+F ext/fts5/fts5_expr.c bcb238ee4ac1164302ab528487520488516bd030
 F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2
 F ext/fts5/fts5_index.c b429e23fabb57506f71e406997cc46b89190dc97
-F ext/fts5/fts5_main.c b4a0fc5bf17f2f1f056ee76cdd7d2af08b360f55
+F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2
 F ext/fts5/fts5_storage.c 3309c6a8e34b974513016fd1ef47c83f5898f94c
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
 F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc
@@ -176,7 +176,7 @@ F ext/fts5/test/fts5porter.test 7cdc07bef301d70eebbfa75dcaf45c3680e1d0e1
 F ext/fts5/test/fts5porter2.test 2e65633d58a1c525d5af0f6c01e5a59155bb3487
 F ext/fts5/test/fts5prefix.test efd42e00bb8e8a36383f25c838185508681c093f
 F ext/fts5/test/fts5query.test f5ec25f5f2fbb70033424113cdffc101b1985a40
-F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab
+F ext/fts5/test/fts5rank.test 3e55e7eeb4c98728e4a3171c9e994e1a2f24eb99
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6
@@ -206,6 +206,7 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
 F ext/misc/amatch.c 211108e201105e4bb0c076527b8cfd34330fc234
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
+F ext/misc/csv.c f01126ba170fd4ef7c752b156568a80c912d4441
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
@@ -215,22 +216,24 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63
 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
 F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a
+F ext/misc/scrub.c ea0903701e3ac02b4466ce9cffd0325c7ddbfcf0
 F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c bf1b922c2750698e9a3d4c50cce6974adb7e93be
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
+F ext/misc/vfsstat.c 6110aeeaab2f1df17a923c8a8acef3c74f6dc515
 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
 F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4
-F ext/rbu/rbu1.test 42bd835e019eff789ec241017965277baeb658b1
+F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 046b0980041d30700464a800bbf6733ed2df515d
 F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702
 F ext/rbu/rbu12.test bde22ed0004dd5d1888c72a84ae407e574aeae16
 F ext/rbu/rbu13.test 462ff799c4afedc3ef8a47ff818c0ffbf14ae4f2
 F ext/rbu/rbu14.test 01f5dcba904aecadbaea69d4ccdc2ea43dd30560
 F ext/rbu/rbu3.test 8bd4c6b87367c358981b6a47dc3d654fa60bff90
-F ext/rbu/rbu5.test 2e24fee3e615aecd99bbdd46967935a641e866f7
+F ext/rbu/rbu5.test cf1fa84d6107efd73e292a7be8e1fb31953ca45c
 F ext/rbu/rbu6.test 32e8ed60631b6facdb6366bd2b5f5f25245e7edb
 F ext/rbu/rbu7.test fd025d5ba440fcfe151fbb0e3835e1e7fe964fa1
 F ext/rbu/rbu8.test 3bbf2c35d71a843c463efe93946f14ad10c3ede0
@@ -238,25 +241,26 @@ F ext/rbu/rbu9.test 0806d1772c9f4981774ff028de6656e4183082af
 F ext/rbu/rbuA.test c1a7b3e2d926b8f8448bb3b4ae787e314ee4b2b3
 F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2
 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831
-F ext/rbu/rbu_common.tcl 0398545fed614f807d5f0ba55a85a51f08ba8f1a
+F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead
 F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695
-F ext/rbu/rbudiff.test 2df0a8a7d998ecf81764c21eeda3cde5611c5091
+F ext/rbu/rbudiff.test 4c9f8df6f723f553781d3d117501b7e9d170a145
 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89
 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca
 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda
 F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a
 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
-F ext/rbu/rbuvacuum.test 66e02cf299836770e718e95c36686be0b26dbda3
-F ext/rbu/sqlite3rbu.c bf36625990c6865ecf08bd844d8097ed2d0a6958
-F ext/rbu/sqlite3rbu.h 2acd0a6344a6079de15c8dc9d84d3df83a665930
-F ext/rbu/test_rbu.c 9bbdf6bd8efd58fbc4f192698df50569598fbb9e
+F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0
+F ext/rbu/rbuvacuum2.test 45009e127c3fb385e5c0fd5a8a63fb922a79d0ab
+F ext/rbu/sqlite3rbu.c 948677ee0ec57da51148e6c5f64ac68afcf36ab2
+F ext/rbu/sqlite3rbu.h db8858120c9be14b60c9225f9da28221f5f6b945
+F ext/rbu/test_rbu.c 9f043b74c46c45b231f4313aed1fccb379a76fe6
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920
+F ext/rtree/rtree.c d26a815b0df1c412a6881dae8d7fd3c9c08cce68
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
-F ext/rtree/rtree1.test 96a80c08440c932cd72aac50660e7af2612d2cda
+F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec
 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba
-F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc
+F ext/rtree/rtree3.test 2cafe8265d1ff28f206fce88d114f208349df482
 F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0
 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e
 F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196
@@ -265,10 +269,11 @@ F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a
 F ext/rtree/rtree9.test b5eb13849545dfd271a54ff16784cb00d8792aea
 F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf
 F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e
-F ext/rtree/rtreeC.test 90aaaffe2fd4f0dcd12289cad5515f6d41f45ffd
+F ext/rtree/rtreeC.test c0a9c67f2efa98b6fae12acb8a28348d231a481d
 F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca
 F ext/rtree/rtreeE.test 45a147a64a76306172819562309681d8e90f94bb
 F ext/rtree/rtreeF.test 66deb9fd1611c7ca2e374adba63debdc2dbb12b4
+F ext/rtree/rtreeG.test 3b185719630795f38594f64cd7d1de86a33f91f1
 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195
 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea
 F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28
@@ -302,7 +307,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 86e6a8d79186ace5ff8b306093da1728d953f796
+F main.mk 3f669c06db5c4a53ff21dda639247c6310497180
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -318,30 +323,30 @@ F src/alter.c 1bb0709b3048e24217b80ec6bd78a3e99a47c01b
 F src/analyze.c 37fedc80ac966ce1745811746e68e4d8fa64c7fe
 F src/attach.c 771153bd1f4ab0b97a44a13dde2c7e5e1efeba22
 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
-F src/backup.c f60f0aa55d25d853ffde53d0b0370a7bb7ee41ce
+F src/backup.c 6df65fdd569c901a418887a1a76f82ec35044556
 F src/bitvec.c 3ee4c8b2c94ed3a7377256e18199e6ff5cf33f63
 F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73
-F src/btree.c 3ae6aea66cc4e13d30162ff0d0d43c7088e34abf
-F src/btree.h a5008b9afe56e8e54ade6c436a910f112defcca9
+F src/btree.c 0621247619df15790f7ed5f14926cfb3925c10b1
+F src/btree.h 2107a2630e02c8cba58bb12ce14e731e734ea29c
 F src/btreeInt.h c18b7d2a3494695133e4e60ee36061d37f45d9a5
-F src/build.c d65be62254ca9df36e1e1c433324f0333f80009c
+F src/build.c d1fdfd7ab8f5447e494ef15825973bf0719527c6
 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
-F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198
+F src/ctime.c 61949e83c4c36e37195a8398ebc752780b534d95
 F src/date.c 1cc9fb516ec9932c6fd4d2a0d2f8bc4480145c39
-F src/dbstat.c c845548d4346e606e2f2b7d2e714ace2b8a7dd1b
+F src/dbstat.c 4f6f7f52b49beb9636ffbd517cfe44a402ba4ad0
 F src/delete.c 4aba4214a377ce8ddde2d2e609777bcc8235200f
-F src/expr.c 8796c0739b2ad091e6779253f62aad6e767e2be1
+F src/expr.c 523a5b1db2b6d88c6eefb224877bf635a3bcfc92
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
-F src/fkey.c 4c0bd09e602b8ae8d36d81e31e4872d0b53c87bb
-F src/func.c ef4c18c8a66143413ce41a58d582d2c14ddf78e1
+F src/fkey.c bc4145347595b7770f9a598cff1c848302cf5413
+F src/func.c 61a4114cf7004f10c542cfabbab9f2bcb9033045
 F src/global.c c45ea22aff29334f6a9ec549235ac3357c970015
 F src/hash.c 55b5fb474100cee0b901edaf203e26c970940f36
 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
 F src/insert.c 8f4e9fcbd8e95e85f15647ba8b413b18d556ec2b
 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e
-F src/loadext.c 8b3a73f0624c5f7cadbd5cb89940783bee1d39a6
+F src/loadext.c 4237fd37ca589f1d90b3ea925dd9df20da8e0c84
 F src/main.c 405d13e3a4f7c5add9fb27702ae70ed0a6e32cca
 F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -362,37 +367,37 @@ F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
 F src/os_unix.c a9443cdab41d7f3cdf0df3a5aab62fd6e1c9b234
-F src/os_win.c 852fc2ff6084296348ed3739c548b2cf32df394e
+F src/os_win.c d4b8faf8896b65818e67070711fdd00d8e620bd6
 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
-F src/pager.c c512873311087cd22372f2b076fc869e090e8e16
-F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681
-F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e
-F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8
-F src/pcache.h 6b865be765d1ebd06145219550b10921c7da7cc9
+F src/pager.c c368634b888b1c8740aea83b36bfd266f2443e60
+F src/pager.h 8ab6b6feeee4bc0439bfde7ee59ba99df98b9bc3
+F src/parse.y 01b9f37c4c7009ab56fda98bc7db4c42643cecfe
+F src/pcache.c 5583c8ade4b05075a60ba953ef471d1c1a9c05df
+F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490
 F src/pcache1.c 7f51d2b541aab57596adf62db2c4bb025d34f04d
-F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84
+F src/pragma.c c8b499756658cb8b82cfdbb5845c22cf11f297aa
 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c
 F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e
 F src/printf.c a5f0ca08ddede803c241266abb46356ec748ded1
 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696
 F src/resolve.c cca3aa77b95706df5d635a2141a4d1de60ae6598
-F src/rowset.c 49eb91c588a2bab36647368e031dc5b66928149d
-F src/select.c fd4a7ce2937497181063cfedb92058ac89491a5d
+F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
+F src/select.c f3c6e9065fb34f6a23af27ec7f1f717ffbfc2ee4
 F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6
-F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54
+F src/sqlite.h.in 5f8113dbec74c6c093ead9930afb8c9fbd9f643d
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 98f72cbfe00169c39089115427d06ea05fe4b4a2
-F src/sqliteInt.h 47a90c85a42210a05387051aaaa661f5c307703d
+F src/sqliteInt.h 2ac73e9fdd138f4607139d9bd6a1e6dcd01837dc
 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247
 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9
-F src/tclsqlite.c 9c4c4589d078de37813ded708d8838b338ffb060
-F src/test1.c c0e5b69f99e95a2c9f55fdb6e46b44f1a15292d8
+F src/tclsqlite.c 25fbbbb97f76dbfd113153fb63f52d7ecfac5dd0
+F src/test1.c 43b37ab2b7338fd3313e74902f0d6c821eae843b
 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b
-F src/test3.c 0df6f8dbb4cbaa7106397c70a271fa6a43659042
+F src/test3.c c75c8af0eadb335236c9e61b51044c58a8f7dd59
 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e
 F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1
-F src/test6.c 2c014d4977efd6107ec9eef3dfdec56ac516f824
+F src/test6.c a684b7abd01352ab50cb79c0bf727e6b3f381a3d
 F src/test7.c 9c89a4f1ed6bb13af0ed805b8d782bd83fcd57e3
 F src/test8.c fa262391d3edea6490a71bfaa8fed477ccbbac75
 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60
@@ -437,30 +442,30 @@ F src/test_windirent.h b12055cab6227f7be10f5c19296f67c60cc5e2a5
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 3b29883b0ce4a6c6f643965b66b5ca6613178e59
-F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373
+F src/treeview.c c56d6ddbed564efda746236b35bcbb8238daac4b
 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280
 F src/update.c 4f05ea8cddfa367d045e03589756c02199e8f9bd
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c 810ec3f22e2d1b62e66c30fe3621ebdedd23584d
 F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52
-F src/vdbe.c 4bad04212736526845fda9ca674afafb10078b2b
+F src/vdbe.c 22b46c3b725e950e9f2760e2d76953d592600ad4
 F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170
 F src/vdbeInt.h ddb157974436d87652de7dc641f7191496d9a8cd
-F src/vdbeapi.c ba85b78fe08dc4a9ce747e62c89a2b4a4547e74c
-F src/vdbeaux.c ace1875da40b7185e604586768d5ac90de7e4f7f
-F src/vdbeblob.c c9f2f494b911c6fa34efd9803f0a10807da80f77
-F src/vdbemem.c 5cfef60e60e19cab6275d1b975bf4c791d575beb
-F src/vdbesort.c 0a8f98366ae794442e6d1ef71d9553226d885d19
+F src/vdbeapi.c 02bcbc2ca5d2004b029088b05b468b394881e103
+F src/vdbeaux.c c90275b0e55a2b32c03dc09314194fe46f2429d8
+F src/vdbeblob.c 83d2d266383157b02e2b809350bb197e89d7895b
+F src/vdbemem.c 1ecaa5ee0caff07255f25d04e8dc88befb6f88d1
+F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c
 F src/vdbetrace.c f75c5455d8cf389ef86a8bfdfd3177e0e3692484
-F src/vtab.c 23b6cdfa996152d43b390504ed4a942c8caf3a00
+F src/vtab.c 948d2d4984219eee37a7bf427d6667e21e6eb92e
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 4db22ed7e77bcf672b1a685d6ddeffba8d5be302
+F src/wal.c 02eeecc265f6ffd0597378f5d8ae9070b62a406a
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c ad8ad50ccf25795f6830b6b864bbfcd060ff08dd
-F src/whereInt.h 3b1fc240e322613ba4e9dc857ca9c7c3390acc74
-F src/wherecode.c d227fca7d766a10fe4ba4d812210f5631f5a98b3
-F src/whereexpr.c eacc0e60d029a082b4fc0cc42ea98544add1319e
+F src/where.c 48eed8ebe319c6cbc7bf7682018f32af0f5189f5
+F src/whereInt.h e5b939701a7ceffc5a3a8188a37f9746416ebcd0
+F src/wherecode.c e20cb381ff621e56a4684c71e31999aca2547ca6
+F src/whereexpr.c c32d47085dbaca0b8fd013210f56693c7d220d48
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -523,6 +528,7 @@ F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f
 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c
 F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee
 F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060
+F test/bestindex3.test b80da904d23581d233a7ceee7d6bbad2b23a8133
 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c
 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59
 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc
@@ -608,8 +614,10 @@ F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d
 F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2
 F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8
 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c
-F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47
+F test/csv01.test 0929a9ce47021519512be92861f29e32d2538e5f
+F test/ctime.test 0b995accd44a52914bd4744d5c1b6e1a56c7897c
 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856
+F test/cursorhint2.test 2b45341d32d1aae9991a00ef31ebca339b274255
 F test/date.test 984ac1e3e5e031386866f034006148d3972b4a65
 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204
 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab
@@ -746,7 +754,7 @@ F test/fts3defer2.test c540f5f5c2840f70c68fd9b597df817ec7170468
 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd
 F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297
 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
-F test/fts3expr.test 3401d47b229c4504424caf362cc4ff704cad4162
+F test/fts3expr.test 9466627007804d855bf9df2a0cfb3dac23686fdc
 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
 F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8
 F test/fts3expr4.test c39a15d676b14fc439d9bf845aa7bddcf4a74dc3
@@ -833,7 +841,7 @@ F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407
 F test/index3.test 81bc47890b8abfb181bc35f8d10b56c069803386
 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6
 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7
-F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a
+F test/index6.test 43b4e29258b978fcdab84fc61df4f5212119dd09
 F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c
 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985
 F test/indexexpr1.test cb71b6586177b840e28110dd952178bb2bdfedc2
@@ -877,6 +885,7 @@ F test/like.test 81632c437a947bf1f7130b19537da6a1a844806a
 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da
 F test/like3.test 3608a2042b6f922f900fbfd5d3ce4e7eca57f7c4
 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e
+F test/limit2.test 55c9f4d08c89311e00afd75045ee1a2aca205cb4
 F test/loadext.test 42a3b8166dfcadcb0e0c8710dc520d97c31a8b98
 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7
 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db
@@ -926,7 +935,7 @@ F test/misc1.test 6430dabfb4b4fa480633590118964201f94d3ccc
 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d
 F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d
 F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3
-F test/misc5.test f96428ea95b3820aafc6f1c50cf48a09e4597ee1
+F test/misc5.test fff0f75e834bc09a39f6079320dd8c37de956f4f
 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91
 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2
 F test/misc8.test 21ac9d35a5e110279ae9e1588b8914f54de1c60b
@@ -977,7 +986,7 @@ F test/parser1.test 222b5cbf3e2e659fec1bf7d723488c8b9c94f1d0
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test b6b3e165fdc1b8c82a820033646dbfc6a7a01746
+F test/permutations.test 544a2f317fc5045bc512ae432f89eae303c0d640
 F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1032,7 +1041,7 @@ F test/select6.test 39eac4a5c03650b2b473c532882273283ee8b7a0
 F test/select7.test 95e370c42d47c3c52377d05e9ffc01ccff7c1f61
 F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d
 F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95
-F test/selectA.test e452bdb975f488ea46d091382a9185b5853ed2c7
+F test/selectA.test 101e722370ac6e84978c2958b8931c78b10a1709
 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25
 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977
 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394
@@ -1053,7 +1062,7 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5
 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e
 F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
-F test/shell1.test 023657b3f8ba108dbd5ff2ebc8fceb3cf6d4ff9f
+F test/shell1.test c90b0415cea6eeefb86b4ab1651b06247922ca52
 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test da513d522ef6f01cee8475dcf8332bff8982b3dd
 F test/shell4.test 69995ee1cc278eb149aa8746ce1f935f4eaf98b9
@@ -1067,7 +1076,7 @@ F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a
 F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5
 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2
 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b
-F test/snapshot.test 3adc4ef09d407b501f899a6c329bdf45dc725c1b
+F test/snapshot.test a19465046168b4420b5afeed37c3214e42a49f4a
 F test/snapshot_fault.test 062ff0438a074978d45e9f9a92e7ad459b74ee73
 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f
 F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087
@@ -1075,7 +1084,7 @@ F test/sort.test c2adc635c2564241fefec0b3a68391ef6868fd3b
 F test/sort2.test cc23b7c19d684657559e8a55b02f7fcee03851d0
 F test/sort3.test 1480ed7c4c157682542224e05e3b75faf4a149e5
 F test/sort4.test 5c34d9623a4ae5921d956dfa2b70e77ed0fc6e5c
-F test/sort5.test e47ec7a490b9b36787755874175d8f413a3883d9
+F test/sort5.test 30cc17768e0c06ecb048e08efec59c11811fd186
 F test/sortfault.test d4ccf606a0c77498e2beb542764fd9394acb4d66
 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452
 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb
@@ -1085,14 +1094,14 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/speedtest1.c 4476f7030775507f060d27b789e1f4123c5a5866
+F test/speedtest1.c 28221f433d358dd3fcf2ca504ed51409aa66ffaf
 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e
 F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142
 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a
-F test/stat.test 66e95f97b9f724f9ab921d054ee0db3c2689f1ee
+F test/stat.test ab95d28503d0f6d98ffd8ce204643c9da090ebf1
 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1
 F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f
 F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f
@@ -1109,12 +1118,13 @@ F test/tabfunc01.test f977868fa8bb7beb4b2072883190411653473906
 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f
 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126
 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930
-F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa
+F test/tclsqlite.test cf0d0a3fd03d64892cec2d48aae9fb2f148680a5
 F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6
 F test/tempdb2.test 4fc92055f2a3f7626c0f2eabbb637dc021b311d5
 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900
 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e
+F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1
 F test/tester.tcl 3ace46dee55f07e28a1e3ab03258b929fefcee5b
 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5
@@ -1352,6 +1362,7 @@ F test/walcksum.test bb234a1bb42248b3515d992b719708015c384278
 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a
 F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36
 F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af
+F test/walcrash4.test 3374d6a0813bfe9d841a973958e0552b545a6423
 F test/walfault.test 1f8389f7709877e9b4cc679033d71d6fe529056b
 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483
 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c
@@ -1365,7 +1376,7 @@ F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
 F test/where.test f0c325563acde44f2c4ea6ba348e9e29f7121757
 F test/where2.test 478d2170637b9211f593120648858593bf2445a1
-F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e
+F test/where3.test 54cdeb02157acc979de41530b804ae7b09552bf1
 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8
 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b
@@ -1410,6 +1421,7 @@ F tool/build-all-msvc.bat 3e4e4043b53f1aede4308e0d2567bbd773614630 x
 F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
 F tool/cg_anno.tcl 692ce4b8693d59e3a3de77ca97f4139ecfa641b0 x
 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2
+F tool/dbhash.c a06228aa21ebc4e6ea8daa486601d938499238a5
 F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2
 F tool/fast_vacuum.c 5ba0d6f5963a0a63bdc42840f678bad75b2ebce1
 F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439
@@ -1417,8 +1429,9 @@ F tool/fuzzershell.c 94019b185caceffc9f7c7b678a6489e42bc2aefa
 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
-F tool/lemon.c f38a55106d79b7a4c063abb958517d6c47dc6ac7
-F tool/lempar.c 404ea3dc27dbeed343f0e61b1d36e97b9f5f0fb6
+F tool/lemon.c 09a96bed19955697a5e20c49ad863ec2005815a2
+F tool/lempar.c 8c4e9d8517e50da391f1d89a519e743dd4afbc09
+F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
 F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7
@@ -1436,6 +1449,7 @@ F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b
 F tool/mkvsix.tcl 4abcaf3267171b2faadaf9b82a0dfbaa6e98f8b7
 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091
 F tool/omittest.tcl 34d7ac01fe4fd18e3637f64abe12c40eca0f6b97
+F tool/opcodesum.tcl 740ed206ba8c5040018988129abbf3089a0ccf4a
 F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b
 F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
@@ -1448,13 +1462,14 @@ F tool/showstat4.c bda40d6e395df7edb6e9ea630784d3d762c35b4b
 F tool/showwal.c ec79959834f7b21f1e0a2aa52bb7c056d2203977
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/spaceanal.tcl 85d90e6674d8298e3eaf82dbcef3abc2d5317f3e
+F tool/speed-check.sh 45d3bf861b009993ff401f0d00e34a4cc937fce4
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
-F tool/sqldiff.c ca315aca4e2d24233e8f2000edea5880c53d1875
+F tool/sqldiff.c 131e9c6e09d3d78e3c4ae8b5fd23451521723aac
 F tool/srcck1.c 4c39bdfa9a92edd20233ee720df84dbeb2417602
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1487,10 +1502,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 14e53d0e2f62d82ae1d64a72fd9711548e3bf5ea
-R 91f64055039542194e221b2aa9a3eada
-T *branch * dev
-T *sym-dev *
+P f81050859170c8708a1b296da8dd3ef0dd314a11
+R 2c74d9aac55f1cbb7c7d3232cf9071e1
+T *branch * rbu-state-api
+T *sym-rbu-state-api *
 T -sym-trunk *
-U drh
-Z 6a1a8543c69edff1da4c7cb6389abc41
+U dan
+Z b7514d1a78fc46b36ed5446f76a12e50

manifest.uuid

@@ -1 +1 @@
-da94a6e0ebacad8f235dcd653a25474327f26137
+92e7df0ff5c4c118c63d92a767dc82700438a310

src/backup.c

@@ -784,8 +784,6 @@ int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){
   rc = sqlite3_backup_finish(&b);
   if( rc==SQLITE_OK ){
     pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED;
-  }else{
-    sqlite3PagerClearCache(sqlite3BtreePager(b.pDest));
   }
 
   assert( sqlite3BtreeIsInTrans(pTo)==0 );

src/btree.c

@@ -450,6 +450,15 @@ static void releasePage(MemPage *pPage);  /* Forward reference */
 static int cursorHoldsMutex(BtCursor *p){
   return sqlite3_mutex_held(p->pBt->mutex);
 }
+
+/* Verify that the cursor and the BtShared agree about what is the current
+** database connetion. This is important in shared-cache mode. If the database 
+** connection pointers get out-of-sync, it is possible for routines like
+** btreeInitPage() to reference an stale connection pointer that references a
+** a connection that has already closed.  This routine is used inside assert()
+** statements only and for the purpose of double-checking that the btree code
+** does keep the database connection pointers up-to-date.
+*/
 static int cursorOwnsBtShared(BtCursor *p){
   assert( cursorHoldsMutex(p) );
   return (p->pBtree->db==p->pBt->db);
@@ -609,21 +618,19 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){
 ** the key.
 */
 static int saveCursorKey(BtCursor *pCur){
-  int rc;
+  int rc = SQLITE_OK;
   assert( CURSOR_VALID==pCur->eState );
   assert( 0==pCur->pKey );
   assert( cursorHoldsMutex(pCur) );
 
-  rc = sqlite3BtreeKeySize(pCur, &pCur->nKey);
-  assert( rc==SQLITE_OK );  /* KeySize() cannot fail */
-
-  /* If this is an intKey table, then the above call to BtreeKeySize()
-  ** stores the integer key in pCur->nKey. In this case this value is
-  ** all that is required. Otherwise, if pCur is not open on an intKey
-  ** table, then malloc space for and store the pCur->nKey bytes of key 
-  ** data.  */
-  if( 0==pCur->curIntKey ){
-    void *pKey = sqlite3Malloc( pCur->nKey );
+  if( pCur->curIntKey ){
+    /* Only the rowid is required for a table btree */
+    pCur->nKey = sqlite3BtreeIntegerKey(pCur);
+  }else{
+    /* For an index btree, save the complete key content */
+    void *pKey;
+    pCur->nKey = sqlite3BtreePayloadSize(pCur);
+    pKey = sqlite3Malloc( pCur->nKey );
     if( pKey ){
       rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey);
       if( rc==SQLITE_OK ){
@@ -1676,11 +1683,11 @@ static int decodeFlags(MemPage *pPage, int flagByte){
   pPage->xCellSize = cellSizePtr;
   pBt = pPage->pBt;
   if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){
-    /* EVIDENCE-OF: R-03640-13415 A value of 5 means the page is an interior
-    ** table b-tree page. */
+    /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an
+    ** interior table b-tree page. */
     assert( (PTF_LEAFDATA|PTF_INTKEY)==5 );
-    /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf
-    ** table b-tree page. */
+    /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a
+    ** leaf table b-tree page. */
     assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 );
     pPage->intKey = 1;
     if( pPage->leaf ){
@@ -1694,11 +1701,11 @@ static int decodeFlags(MemPage *pPage, int flagByte){
     pPage->maxLocal = pBt->maxLeaf;
     pPage->minLocal = pBt->minLeaf;
   }else if( flagByte==PTF_ZERODATA ){
-    /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior
-    ** index b-tree page. */
+    /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an
+    ** interior index b-tree page. */
     assert( (PTF_ZERODATA)==2 );
-    /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf
-    ** index b-tree page. */
+    /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a
+    ** leaf index b-tree page. */
     assert( (PTF_ZERODATA|PTF_LEAF)==10 );
     pPage->intKey = 0;
     pPage->intKeyLeaf = 0;
@@ -4262,46 +4269,33 @@ int sqlite3BtreeCursorIsValid(BtCursor *pCur){
 #endif /* NDEBUG */
 
 /*
-** Set *pSize to the size of the buffer needed to hold the value of
-** the key for the current entry.  If the cursor is not pointing
-** to a valid entry, *pSize is set to 0. 
-**
-** For a table with the INTKEY flag set, this routine returns the key
-** itself, not the number of bytes in the key.
-**
-** The caller must position the cursor prior to invoking this routine.
-** 
-** This routine cannot fail.  It always returns SQLITE_OK.  
+** Return the value of the integer key or "rowid" for a table btree.
+** This routine is only valid for a cursor that is pointing into a
+** ordinary table btree.  If the cursor points to an index btree or
+** is invalid, the result of this routine is undefined.
 */
-int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){
+i64 sqlite3BtreeIntegerKey(BtCursor *pCur){
   assert( cursorHoldsMutex(pCur) );
   assert( pCur->eState==CURSOR_VALID );
+  assert( pCur->curIntKey );
   getCellInfo(pCur);
-  *pSize = pCur->info.nKey;
-  return SQLITE_OK;
+  return pCur->info.nKey;
 }
 
 /*
-** Set *pSize to the number of bytes of data in the entry the
-** cursor currently points to.
+** Return the number of bytes of payload for the entry that pCur is
+** currently pointing to.  For table btrees, this will be the amount
+** of data.  For index btrees, this will be the size of the key.
 **
 ** The caller must guarantee that the cursor is pointing to a non-NULL
 ** valid entry.  In other words, the calling procedure must guarantee
 ** that the cursor has Cursor.eState==CURSOR_VALID.
-**
-** Failure is not possible.  This function always returns SQLITE_OK.
-** It might just as well be a procedure (returning void) but we continue
-** to return an integer result code for historical reasons.
 */
-int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){
-  assert( cursorOwnsBtShared(pCur) );
+u32 sqlite3BtreePayloadSize(BtCursor *pCur){
+  assert( cursorHoldsMutex(pCur) );
   assert( pCur->eState==CURSOR_VALID );
-  assert( pCur->iPage>=0 );
-  assert( pCur->iPage<BTCURSOR_MAX_DEPTH );
-  assert( pCur->apPage[pCur->iPage]->intKeyLeaf==1 );
   getCellInfo(pCur);
-  *pSize = pCur->info.nPayload;
-  return SQLITE_OK;
+  return pCur->info.nPayload;
 }
 
 /*
@@ -4743,10 +4737,7 @@ static const void *fetchPayload(
 ** These routines is used to get quick access to key and data
 ** in the common case where no overflow pages are used.
 */
-const void *sqlite3BtreeKeyFetch(BtCursor *pCur, u32 *pAmt){
-  return fetchPayload(pCur, pAmt);
-}
-const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){
+const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){
   return fetchPayload(pCur, pAmt);
 }
 
@@ -5079,11 +5070,12 @@ int sqlite3BtreeMovetoUnpacked(
   assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
   assert( pRes );
   assert( (pIdxKey==0)==(pCur->pKeyInfo==0) );
+  assert( pCur->eState!=CURSOR_VALID || (pIdxKey==0)==(pCur->curIntKey!=0) );
 
   /* If the cursor is already positioned at the point we are trying
   ** to move to, then just return without doing any work */
-  if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0
-   && pCur->curIntKey 
+  if( pIdxKey==0
+   && pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0
   ){
     if( pCur->info.nKey==intKey ){
       *pRes = 0;
@@ -6072,9 +6064,7 @@ static int clearCell(
 static int fillInCell(
   MemPage *pPage,                /* The page that contains the cell */
   unsigned char *pCell,          /* Complete text of the cell */
-  const void *pKey, i64 nKey,    /* The key */
-  const void *pData,int nData,   /* The data */
-  int nZero,                     /* Extra zero bytes to append to pData */
+  const BtreePayload *pX,        /* Payload with which to construct the cell */
   int *pnSize                    /* Write cell size here */
 ){
   int nPayload;
@@ -6098,26 +6088,23 @@ static int fillInCell(
 
   /* Fill in the header. */
   nHeader = pPage->childPtrSize;
-  nPayload = nData + nZero;
-  if( pPage->intKeyLeaf ){
-    nHeader += putVarint32(&pCell[nHeader], nPayload);
-  }else{
-    assert( nData==0 );
-    assert( nZero==0 );
-  }
-  nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey);
-  
-  /* Fill in the payload size */
   if( pPage->intKey ){
-    pSrc = pData;
-    nSrc = nData;
-    nData = 0;
-  }else{ 
-    assert( nKey<=0x7fffffff && pKey!=0 );
-    nPayload = (int)nKey;
-    pSrc = pKey;
-    nSrc = (int)nKey;
+    nPayload = pX->nData + pX->nZero;
+    pSrc = pX->pData;
+    nSrc = pX->nData;
+    assert( pPage->intKeyLeaf ); /* fillInCell() only called for leaves */
+    nHeader += putVarint32(&pCell[nHeader], nPayload);
+    nHeader += putVarint(&pCell[nHeader], *(u64*)&pX->nKey);
+  }else{
+    assert( pX->nData==0 );
+    assert( pX->nZero==0 );
+    assert( pX->nKey<=0x7fffffff && pX->pKey!=0 );
+    nSrc = nPayload = (int)pX->nKey;
+    pSrc = pX->pKey;
+    nHeader += putVarint32(&pCell[nHeader], nPayload);
   }
+  
+  /* Fill in the payload */
   if( nPayload<=pPage->maxLocal ){
     n = nHeader + nPayload;
     testcase( n==3 );
@@ -6155,7 +6142,7 @@ static int fillInCell(
     CellInfo info;
     pPage->xParseCell(pPage, pCell, &info);
     assert( nHeader==(int)(info.pPayload - pCell) );
-    assert( info.nKey==nKey );
+    assert( info.nKey==pX->nKey );
     assert( *pnSize == info.nSize );
     assert( spaceLeft == info.nLocal );
   }
@@ -6240,10 +6227,6 @@ static int fillInCell(
     pSrc += n;
     nSrc -= n;
     spaceLeft -= n;
-    if( nSrc==0 ){
-      nSrc = nData;
-      pSrc = pData;
-    }
   }
   releasePage(pToRelease);
   return SQLITE_OK;
@@ -6310,6 +6293,8 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){
 ** in pTemp or the original pCell) and also record its index. 
 ** Allocating a new entry in pPage->aCell[] implies that 
 ** pPage->nOverflow is incremented.
+**
+** *pRC must be SQLITE_OK when this routine is called.
 */
 static void insertCell(
   MemPage *pPage,   /* Page into which we are copying */
@@ -6325,8 +6310,7 @@ static void insertCell(
   u8 *data;         /* The content of the whole page */
   u8 *pIns;         /* The point in pPage->aCellIdx[] where no cell inserted */
 
-  if( *pRC ) return;
-
+  assert( *pRC==SQLITE_OK );
   assert( i>=0 && i<=pPage->nCell+pPage->nOverflow );
   assert( MX_CELL(pPage->pBt)<=10921 );
   assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB );
@@ -6832,8 +6816,10 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){
     while( ((*(pOut++) = *(pCell++))&0x80) && pCell<pStop );
 
     /* Insert the new divider cell into pParent. */
-    insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace),
-               0, pPage->pgno, &rc);
+    if( rc==SQLITE_OK ){
+      insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace),
+                   0, pPage->pgno, &rc);
+    }
 
     /* Set the right-child pointer of pParent to point to the new page. */
     put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew);
@@ -7925,13 +7911,19 @@ static int balance(BtCursor *pCur){
 
 
 /*
-** Insert a new record into the BTree.  The key is given by (pKey,nKey)
-** and the data is given by (pData,nData).  The cursor is used only to
-** define what table the record should be inserted into.  The cursor
-** is left pointing at a random location.
+** Insert a new record into the BTree.  The content of the new record
+** is described by the pX object.  The pCur cursor is used only to
+** define what table the record should be inserted into, and is left
+** pointing at a random location.
 **
-** For an INTKEY table, only the nKey value of the key is used.  pKey is
-** ignored.  For a ZERODATA table, the pData and nData are both ignored.
+** For a table btree (used for rowid tables), only the pX.nKey value of
+** the key is used. The pX.pKey value must be NULL.  The pX.nKey is the
+** rowid or INTEGER PRIMARY KEY of the row.  The pX.nData,pData,nZero fields
+** hold the content of the row.
+**
+** For an index btree (used for indexes and WITHOUT ROWID tables), the
+** key is an arbitrary byte sequence stored in pX.pKey,nKey.  The 
+** pX.pData,nData,nZero fields must be zero.
 **
 ** If the seekResult parameter is non-zero, then a successful call to
 ** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already
@@ -7948,9 +7940,7 @@ static int balance(BtCursor *pCur){
 */
 int sqlite3BtreeInsert(
   BtCursor *pCur,                /* Insert data into the table of this cursor */
-  const void *pKey, i64 nKey,    /* The key of the new record */
-  const void *pData, int nData,  /* The data of the new record */
-  int nZero,                     /* Number of extra 0 bytes to append to data */
+  const BtreePayload *pX,        /* Content of the row to be inserted */
   int appendBias,                /* True if this is likely an append */
   int seekResult                 /* Result of prior MovetoUnpacked() call */
 ){
@@ -7980,7 +7970,7 @@ int sqlite3BtreeInsert(
   ** keys with no associated data. If the cursor was opened expecting an
   ** intkey table, the caller should be inserting integer keys with a
   ** blob of associated data.  */
-  assert( (pKey==0)==(pCur->pKeyInfo==0) );
+  assert( (pX->pKey==0)==(pCur->pKeyInfo==0) );
 
   /* Save the positions of any other cursors open on this table.
   **
@@ -7999,38 +7989,38 @@ int sqlite3BtreeInsert(
   }
 
   if( pCur->pKeyInfo==0 ){
-    assert( pKey==0 );
+    assert( pX->pKey==0 );
     /* If this is an insert into a table b-tree, invalidate any incrblob 
     ** cursors open on the row being replaced */
-    invalidateIncrblobCursors(p, nKey, 0);
+    invalidateIncrblobCursors(p, pX->nKey, 0);
 
     /* If the cursor is currently on the last row and we are appending a
     ** new row onto the end, set the "loc" to avoid an unnecessary
     ** btreeMoveto() call */
-    if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0
-      && pCur->info.nKey==nKey-1 ){
+    if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0
+      && pCur->info.nKey==pX->nKey-1 ){
        loc = -1;
     }else if( loc==0 ){
-      rc = sqlite3BtreeMovetoUnpacked(pCur, 0, nKey, appendBias, &loc);
+      rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, appendBias, &loc);
       if( rc ) return rc;
     }
   }else if( loc==0 ){
-    rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc);
+    rc = btreeMoveto(pCur, pX->pKey, pX->nKey, appendBias, &loc);
     if( rc ) return rc;
   }
   assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) );
 
   pPage = pCur->apPage[pCur->iPage];
-  assert( pPage->intKey || nKey>=0 );
+  assert( pPage->intKey || pX->nKey>=0 );
   assert( pPage->leaf || !pPage->intKey );
 
   TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n",
-          pCur->pgnoRoot, nKey, nData, pPage->pgno,
+          pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno,
           loc==0 ? "overwrite" : "new entry"));
   assert( pPage->isInit );
   newCell = pBt->pTmpSpace;
   assert( newCell!=0 );
-  rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew);
+  rc = fillInCell(pPage, newCell, pX, &szNew);
   if( rc ) goto end_insert;
   assert( szNew==pPage->xCellSize(pPage, newCell) );
   assert( szNew <= MX_CELL_SIZE(pBt) );
@@ -8056,6 +8046,7 @@ int sqlite3BtreeInsert(
     assert( pPage->leaf );
   }
   insertCell(pPage, idx, newCell, szNew, 0, 0, &rc);
+  assert( pPage->nOverflow==0 || rc==SQLITE_OK );
   assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 );
 
   /* If no error has occurred and pPage has an overflow cell, call balance() 
@@ -8079,7 +8070,8 @@ int sqlite3BtreeInsert(
   ** row without seeking the cursor. This can be a big performance boost.
   */
   pCur->info.nSize = 0;
-  if( rc==SQLITE_OK && pPage->nOverflow ){
+  if( pPage->nOverflow ){
+    assert( rc==SQLITE_OK );
     pCur->curFlags &= ~(BTCF_ValidNKey);
     rc = balance(pCur);
 
@@ -8215,7 +8207,9 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){
     pTmp = pBt->pTmpSpace;
     assert( pTmp!=0 );
     rc = sqlite3PagerWrite(pLeaf->pDbPage);
-    insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc);
+    if( rc==SQLITE_OK ){
+      insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc);
+    }
     dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc);
     if( rc ) return rc;
   }

src/btree.h

@@ -39,6 +39,7 @@
 typedef struct Btree Btree;
 typedef struct BtCursor BtCursor;
 typedef struct BtShared BtShared;
+typedef struct BtreePayload BtreePayload;
 
 
 int sqlite3BtreeOpen(
@@ -250,19 +251,43 @@ int sqlite3BtreeDelete(BtCursor*, u8 flags);
 #define BTREE_SAVEPOSITION 0x02  /* Leave cursor pointing at NEXT or PREV */
 #define BTREE_AUXDELETE    0x04  /* not the primary delete operation */
 
-int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey,
-                                  const void *pData, int nData,
-                                  int nZero, int bias, int seekResult);
+/* An instance of the BtreePayload object describes the content of a single
+** entry in either an index or table btree.
+**
+** Index btrees (used for indexes and also WITHOUT ROWID tables) contain
+** an arbitrary key and no data.  These btrees have pKey,nKey set to their
+** key and pData,nData,nZero set to zero.
+**
+** Table btrees (used for rowid tables) contain an integer rowid used as
+** the key and passed in the nKey field.  The pKey field is zero.  
+** pData,nData hold the content of the new entry.  nZero extra zero bytes
+** are appended to the end of the content when constructing the entry.
+**
+** This object is used to pass information into sqlite3BtreeInsert().  The
+** same information used to be passed as five separate parameters.  But placing
+** the information into this object helps to keep the interface more 
+** organized and understandable, and it also helps the resulting code to
+** run a little faster by using fewer registers for parameter passing.
+*/
+struct BtreePayload {
+  const void *pKey;       /* Key content for indexes.  NULL for tables */
+  sqlite3_int64 nKey;     /* Size of pKey for indexes.  PRIMARY KEY for tabs */
+  const void *pData;      /* Data for tables.  NULL for indexes */
+  int nData;              /* Size of pData.  0 if none. */
+  int nZero;              /* Extra zero data appended after pData,nData */
+};
+
+int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload,
+                       int bias, int seekResult);
 int sqlite3BtreeFirst(BtCursor*, int *pRes);
 int sqlite3BtreeLast(BtCursor*, int *pRes);
 int sqlite3BtreeNext(BtCursor*, int *pRes);
 int sqlite3BtreeEof(BtCursor*);
 int sqlite3BtreePrevious(BtCursor*, int *pRes);
-int sqlite3BtreeKeySize(BtCursor*, i64 *pSize);
+i64 sqlite3BtreeIntegerKey(BtCursor*);
 int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*);
-const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt);
-const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt);
-int sqlite3BtreeDataSize(BtCursor*, u32 *pSize);
+const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt);
+u32 sqlite3BtreePayloadSize(BtCursor*);
 int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*);
 
 char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*);

src/build.c

@@ -338,7 +338,7 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){
 */
 Table *sqlite3LocateTable(
   Parse *pParse,         /* context in which to report errors */
-  int isView,            /* True if looking for a VIEW rather than a TABLE */
+  u32 flags,             /* LOCATE_VIEW or LOCATE_NOERR */
   const char *zName,     /* Name of the table we are looking for */
   const char *zDbase     /* Name of the database.  Might be NULL */
 ){
@@ -352,7 +352,7 @@ Table *sqlite3LocateTable(
 
   p = sqlite3FindTable(pParse->db, zName, zDbase);
   if( p==0 ){
-    const char *zMsg = isView ? "no such view" : "no such table";
+    const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table";
 #ifndef SQLITE_OMIT_VIRTUALTABLE
     if( sqlite3FindDbName(pParse->db, zDbase)<1 ){
       /* If zName is the not the name of a table in the schema created using
@@ -364,12 +364,14 @@ Table *sqlite3LocateTable(
       }
     }
 #endif
-    if( zDbase ){
-      sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName);
-    }else{
-      sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName);
+    if( (flags & LOCATE_NOERR)==0 ){
+      if( zDbase ){
+        sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName);
+      }else{
+        sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName);
+      }
+      pParse->checkSchema = 1;
     }
-    pParse->checkSchema = 1;
   }
 
   return p;
@@ -386,7 +388,7 @@ Table *sqlite3LocateTable(
 */
 Table *sqlite3LocateTableItem(
   Parse *pParse, 
-  int isView, 
+  u32 flags,
   struct SrcList_item *p
 ){
   const char *zDb;
@@ -397,7 +399,7 @@ Table *sqlite3LocateTableItem(
   }else{
     zDb = p->zDatabase;
   }
-  return sqlite3LocateTable(pParse, isView, p->zName, zDb);
+  return sqlite3LocateTable(pParse, flags, p->zName, zDb);
 }
 
 /*
@@ -605,8 +607,9 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
   /* Delete all indices associated with this table. */
   for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){
     pNext = pIndex->pNext;
-    assert( pIndex->pSchema==pTable->pSchema );
-    if( !db || db->pnBytesFreed==0 ){
+    assert( pIndex->pSchema==pTable->pSchema
+         || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) );
+    if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){
       char *zName = pIndex->zName; 
       TESTONLY ( Index *pOld = ) sqlite3HashInsert(
          &pIndex->pSchema->idxHash, zName, 0
@@ -1288,7 +1291,7 @@ void sqlite3AddPrimaryKey(
   Column *pCol = 0;
   int iCol = -1, i;
   int nTerm;
-  if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit;
+  if( pTab==0 ) goto primary_key_exit;
   if( pTab->tabFlags & TF_HasPrimaryKey ){
     sqlite3ErrorMsg(pParse, 
       "table \"%s\" has more than one primary key", pTab->zName);
@@ -1334,12 +1337,8 @@ void sqlite3AddPrimaryKey(
        "INTEGER PRIMARY KEY");
 #endif
   }else{
-    Index *p;
-    p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0,
-                           0, sortOrder, 0);
-    if( p ){
-      p->idxType = SQLITE_IDXTYPE_PRIMARYKEY;
-    }
+    sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0,
+                           0, sortOrder, 0, SQLITE_IDXTYPE_PRIMARYKEY);
     pList = 0;
   }
 
@@ -1656,21 +1655,23 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){
 ** are appropriate for a WITHOUT ROWID table instead of a rowid table.
 ** Changes include:
 **
-**     (1)  Convert the OP_CreateTable into an OP_CreateIndex.  There is
+**     (1)  Set all columns of the PRIMARY KEY schema object to be NOT NULL.
+**     (2)  Convert the OP_CreateTable into an OP_CreateIndex.  There is
 **          no rowid btree for a WITHOUT ROWID.  Instead, the canonical
 **          data storage is a covering index btree.
-**     (2)  Bypass the creation of the sqlite_master table entry
+**     (3)  Bypass the creation of the sqlite_master table entry
 **          for the PRIMARY KEY as the primary key index is now
 **          identified by the sqlite_master table entry of the table itself.
-**     (3)  Set the Index.tnum of the PRIMARY KEY Index object in the
+**     (4)  Set the Index.tnum of the PRIMARY KEY Index object in the
 **          schema to the rootpage from the main table.
-**     (4)  Set all columns of the PRIMARY KEY schema object to be NOT NULL.
 **     (5)  Add all table columns to the PRIMARY KEY Index object
 **          so that the PRIMARY KEY is a covering index.  The surplus
 **          columns are part of KeyInfo.nXField and are not used for
 **          sorting or lookup or uniqueness checks.
 **     (6)  Replace the rowid tail on all automatically generated UNIQUE
 **          indices with the PRIMARY KEY columns.
+**
+** For virtual tables, only (1) is performed.
 */
 static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
   Index *pIdx;
@@ -1680,6 +1681,20 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
   sqlite3 *db = pParse->db;
   Vdbe *v = pParse->pVdbe;
 
+  /* Mark every PRIMARY KEY column as NOT NULL (except for imposter tables)
+  */
+  if( !db->init.imposterTable ){
+    for(i=0; i<pTab->nCol; i++){
+      if( (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 ){
+        pTab->aCol[i].notNull = OE_Abort;
+      }
+    }
+  }
+
+  /* The remaining transformations only apply to b-tree tables, not to
+  ** virtual tables */
+  if( IN_DECLARE_VTAB ) return;
+
   /* Convert the OP_CreateTable opcode that would normally create the
   ** root-page for the table into an OP_CreateIndex opcode.  The index
   ** created will become the PRIMARY KEY index.
@@ -1701,9 +1716,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
     if( pList==0 ) return;
     pList->a[0].sortOrder = pParse->iPkSortOrder;
     assert( pParse->pNewTable==pTab );
-    pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0);
-    if( pPk==0 ) return;
-    pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY;
+    sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0,
+                       SQLITE_IDXTYPE_PRIMARYKEY);
+    if( db->mallocFailed ) return;
+    pPk = sqlite3PrimaryKeyIndex(pTab);
     pTab->iPKey = -1;
   }else{
     pPk = sqlite3PrimaryKeyIndex(pTab);
@@ -1731,19 +1747,11 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
     }
     pPk->nKeyCol = j;
   }
-  pPk->isCovering = 1;
   assert( pPk!=0 );
+  pPk->isCovering = 1;
+  if( !db->init.imposterTable ) pPk->uniqNotNull = 1;
   nPk = pPk->nKeyCol;
 
-  /* Make sure every column of the PRIMARY KEY is NOT NULL.  (Except,
-  ** do not enforce this for imposter tables.) */
-  if( !db->init.imposterTable ){
-    for(i=0; i<nPk; i++){
-      pTab->aCol[pPk->aiColumn[i]].notNull = OE_Abort;
-    }
-    pPk->uniqNotNull = 1;
-  }
-
   /* The root page of the PRIMARY KEY is the table root page */
   pPk->tnum = pTab->tnum;
 
@@ -2498,6 +2506,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){
   assert( pName->nSrc==1 );
   if( sqlite3ReadSchema(pParse) ) goto exit_drop_table;
   if( noErr ) db->suppressErr++;
+  assert( isView==0 || isView==LOCATE_VIEW );
   pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]);
   if( noErr ) db->suppressErr--;
 
@@ -2868,12 +2877,8 @@ Index *sqlite3AllocateIndexObject(
 ** pList is a list of columns to be indexed.  pList will be NULL if this
 ** is a primary key or unique-constraint on the most recent column added
 ** to the table currently under construction.  
-**
-** If the index is created successfully, return a pointer to the new Index
-** structure. This is used by sqlite3AddPrimaryKey() to mark the index
-** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY)
 */
-Index *sqlite3CreateIndex(
+void sqlite3CreateIndex(
   Parse *pParse,     /* All information about this parse */
   Token *pName1,     /* First part of index name. May be NULL */
   Token *pName2,     /* Second part of index name. May be NULL */
@@ -2883,9 +2888,9 @@ Index *sqlite3CreateIndex(
   Token *pStart,     /* The CREATE token that begins this statement */
   Expr *pPIWhere,    /* WHERE clause for partial indices */
   int sortOrder,     /* Sort order of primary key when pList==NULL */
-  int ifNotExist     /* Omit error if index already exists */
+  int ifNotExist,    /* Omit error if index already exists */
+  u8 idxType         /* The index type */
 ){
-  Index *pRet = 0;     /* Pointer to return */
   Table *pTab = 0;     /* Table to be indexed */
   Index *pIndex = 0;   /* The index to be created */
   char *zName = 0;     /* Name of the index */
@@ -2903,7 +2908,10 @@ Index *sqlite3CreateIndex(
   char *zExtra = 0;                /* Extra space after the Index object */
   Index *pPk = 0;      /* PRIMARY KEY index for WITHOUT ROWID tables */
 
-  if( db->mallocFailed || IN_DECLARE_VTAB || pParse->nErr>0 ){
+  if( db->mallocFailed || pParse->nErr>0 ){
+    goto exit_create_index;
+  }
+  if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){
     goto exit_create_index;
   }
   if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
@@ -3092,7 +3100,7 @@ Index *sqlite3CreateIndex(
   pIndex->pTable = pTab;
   pIndex->onError = (u8)onError;
   pIndex->uniqNotNull = onError!=OE_None;
-  pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE;
+  pIndex->idxType = idxType;
   pIndex->pSchema = db->aDb[iDb].pSchema;
   pIndex->nKeyCol = pList->nExpr;
   if( pPIWhere ){
@@ -3272,7 +3280,7 @@ Index *sqlite3CreateIndex(
             pIdx->onError = pIndex->onError;
           }
         }
-        pRet = pIdx;
+        if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType;
         goto exit_create_index;
       }
     }
@@ -3284,6 +3292,7 @@ Index *sqlite3CreateIndex(
   assert( pParse->nErr==0 );
   if( db->init.busy ){
     Index *p;
+    assert( !IN_DECLARE_VTAB );
     assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) );
     p = sqlite3HashInsert(&pIndex->pSchema->idxHash, 
                           pIndex->zName, pIndex);
@@ -3365,7 +3374,7 @@ Index *sqlite3CreateIndex(
       sqlite3ChangeCookie(pParse, iDb);
       sqlite3VdbeAddParseSchemaOp(v, iDb,
          sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName));
-      sqlite3VdbeAddOp1(v, OP_Expire, 0);
+      sqlite3VdbeAddOp0(v, OP_Expire);
     }
 
     sqlite3VdbeJumpHere(v, pIndex->tnum);
@@ -3390,7 +3399,6 @@ Index *sqlite3CreateIndex(
       pIndex->pNext = pOther->pNext;
       pOther->pNext = pIndex;
     }
-    pRet = pIndex;
     pIndex = 0;
   }
 
@@ -3401,7 +3409,6 @@ exit_create_index:
   sqlite3ExprListDelete(db, pList);
   sqlite3SrcListDelete(db, pTblName);
   sqlite3DbFree(db, zName);
-  return pRet;
 }
 
 /*
@@ -3430,10 +3437,11 @@ void sqlite3DefaultRowEst(Index *pIdx){
   int i;
 
   /* Set the first entry (number of rows in the index) to the estimated 
-  ** number of rows in the table. Or 10, if the estimated number of rows 
-  ** in the table is less than that.  */
+  ** number of rows in the table, or half the number of rows in the table
+  ** for a partial index.   But do not let the estimate drop below 10. */
   a[0] = pIdx->pTable->nRowLogEst;
-  if( a[0]<33 ) a[0] = 33;        assert( 33==sqlite3LogEst(10) );
+  if( pIdx->pPartIdxWhere!=0 ) a[0] -= 10;  assert( 10==sqlite3LogEst(2) );
+  if( a[0]<33 ) a[0] = 33;                  assert( 33==sqlite3LogEst(10) );
 
   /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is
   ** 6 and each subsequent value (if any) is 5.  */
@@ -4315,10 +4323,6 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){
 /*
 ** Return a KeyInfo structure that is appropriate for the given Index.
 **
-** The KeyInfo structure for an index is cached in the Index object.
-** So there might be multiple references to the returned pointer.  The
-** caller should not try to modify the KeyInfo object.
-**
 ** The caller should invoke sqlite3KeyInfoUnref() on the returned object
 ** when it has finished using it.
 */

src/ctime.c

@@ -45,6 +45,13 @@ static const char * const azCompileOpt[] = {
 #if SQLITE_CHECK_PAGES
   "CHECK_PAGES",
 #endif
+#if defined(__clang__) && defined(__clang_version__)
+  "COMPILER=clang-" __clang_version__,
+#elif defined(_MSC_VER)
+  "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER),
+#elif defined(__GNUC__) && defined(__VERSION__)
+  "COMPILER=gcc-" __VERSION__,
+#endif
 #if SQLITE_COVERAGE_TEST
   "COVERAGE_TEST",
 #endif
@@ -64,7 +71,7 @@ static const char * const azCompileOpt[] = {
   "DISABLE_LFS",
 #endif
 #if SQLITE_ENABLE_8_3_NAMES
-  "ENABLE_8_3_NAMES",
+  "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES),
 #endif
 #if SQLITE_ENABLE_API_ARMOR
   "ENABLE_API_ARMOR",

src/dbstat.c

@@ -58,10 +58,10 @@
 */
 #define VTAB_SCHEMA                                                         \
   "CREATE TABLE xx( "                                                       \
-  "  name       STRING,           /* Name of table or index */"             \
-  "  path       INTEGER,          /* Path to page from root */"             \
+  "  name       TEXT,             /* Name of table or index */"             \
+  "  path       TEXT,             /* Path to page from root */"             \
   "  pageno     INTEGER,          /* Page number */"                        \
-  "  pagetype   STRING,           /* 'internal', 'leaf' or 'overflow' */"   \
+  "  pagetype   TEXT,             /* 'internal', 'leaf' or 'overflow' */"   \
   "  ncell      INTEGER,          /* Cells on page (0 for overflow) */"     \
   "  payload    INTEGER,          /* Bytes of payload on this page */"      \
   "  unused     INTEGER,          /* Bytes of unused space on this page */" \

src/expr.c

@@ -1818,6 +1818,11 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){
           eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0];
 
           if( prRhsHasNull && !pTab->aCol[iCol].notNull ){
+#ifdef SQLITE_ENABLE_COLUMN_USED_MASK
+            const i64 sOne = 1;
+            sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, 
+                iTab, 0, 0, (u8*)&sOne, P4_INT64);
+#endif
             *prRhsHasNull = ++pParse->nMem;
             sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull);
           }
@@ -2222,8 +2227,7 @@ static void sqlite3ExprCodeIN(
     if( eType==IN_INDEX_ROWID ){
       /* In this case, the RHS is the ROWID of table b-tree
       */
-      sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v);
-      sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1);
+      sqlite3VdbeAddOp3(v, OP_SeekRowid, pExpr->iTable, destIfFalse, r1);
       VdbeCoverage(v);
     }else{
       /* In this case, the RHS is an index b-tree.
@@ -2536,7 +2540,7 @@ void sqlite3ExprCodeGetColumnOfTable(
   }else{
     int op = IsVirtual(pTab) ? OP_VColumn : OP_Column;
     int x = iCol;
-    if( !HasRowid(pTab) ){
+    if( !HasRowid(pTab) && !IsVirtual(pTab) ){
       x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol);
     }
     sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut);

src/fkey.c

@@ -1372,7 +1372,8 @@ void sqlite3FkDelete(sqlite3 *db, Table *pTab){
   FKey *pFKey;                    /* Iterator variable */
   FKey *pNext;                    /* Copy of pFKey->pNextFrom */
 
-  assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) );
+  assert( db==0 || IsVirtual(pTab)
+         || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) );
   for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){
 
     /* Remove the FK from the fkeyHash hash table. */

src/func.c

@@ -740,7 +740,7 @@ static int patternCompare(
     }
     c2 = Utf8Read(zString);
     if( c==c2 ) continue;
-    if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){
+    if( noCase  && sqlite3Tolower(c)==sqlite3Tolower(c2) && c<0x80 && c2<0x80 ){
       continue;
     }
     if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue;

src/loadext.c

@@ -446,6 +446,7 @@ static int sqlite3LoadExtension(
   void **aHandle;
   u64 nMsg = 300 + sqlite3Strlen30(zFile);
   int ii;
+  int rc;
 
   /* Shared library endings to try if zFile cannot be loaded as written */
   static const char *azEndings[] = {
@@ -548,7 +549,9 @@ static int sqlite3LoadExtension(
     return SQLITE_ERROR;
   }
   sqlite3_free(zAltEntry);
-  if( xInit(db, &zErrmsg, &sqlite3Apis) ){
+  rc = xInit(db, &zErrmsg, &sqlite3Apis);
+  if( rc ){
+    if( rc==SQLITE_OK_LOAD_PERMANENTLY ) return SQLITE_OK;
     if( pzErrMsg ){
       *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg);
     }

src/os_win.c

@@ -5269,6 +5269,18 @@ static int winFullPathname(
   int nFull,                    /* Size of output buffer in bytes */
   char *zFull                   /* Output buffer */
 ){
+#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__)
+  DWORD nByte;
+  void *zConverted;
+  char *zOut;
+#endif
+
+  /* If this path name begins with "/X:", where "X" is any alphabetic
+  ** character, discard the initial "/" from the pathname.
+  */
+  if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){
+    zRelative++;
+  }
 
 #if defined(__CYGWIN__)
   SimulateIOError( return SQLITE_ERROR );
@@ -5347,17 +5359,6 @@ static int winFullPathname(
 #endif
 
 #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__)
-  DWORD nByte;
-  void *zConverted;
-  char *zOut;
-
-  /* If this path name begins with "/X:", where "X" is any alphabetic
-  ** character, discard the initial "/" from the pathname.
-  */
-  if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){
-    zRelative++;
-  }
-
   /* It's odd to simulate an io-error here, but really this is just
   ** using the io-error infrastructure to test that SQLite handles this
   ** function failing. This function could fail if, for example, the

src/pager.c

@@ -1876,20 +1876,24 @@ static int pager_error(Pager *pPager, int rc){
 static int pager_truncate(Pager *pPager, Pgno nPage);
 
 /*
-** The write transaction open on the pager passed as the only argument is
-** being committed. This function returns true if all dirty pages should
-** be flushed to disk, or false otherwise. Pages should be flushed to disk
-** unless one of the following is true:
+** The write transaction open on pPager is being committed (bCommit==1)
+** or rolled back (bCommit==0).
 **
-**   * The db is an in-memory database.
+** Return TRUE if and only if all dirty pages should be flushed to disk.
 **
-**   * The db is a temporary database and the db file has not been opened.
+** Rules:
 **
-**   * The db is a temporary database and the cache contains less than
-**     C/4 dirty pages, where C is the configured cache-size.
+**   *  For non-TEMP databases, always sync to disk.  This is necessary
+**      for transactions to be durable.
+**
+**   *  Sync TEMP database only on a COMMIT (not a ROLLBACK) when the backing
+**      file has been created already (via a spill on pagerStress()) and
+**      when the number of dirty pages in memory exceeds 25% of the total
+**      cache size.
 */
-static int pagerFlushOnCommit(Pager *pPager){
+static int pagerFlushOnCommit(Pager *pPager, int bCommit){
   if( pPager->tempFile==0 ) return 1;
+  if( !bCommit ) return 0;
   if( !isOpen(pPager->fd) ) return 0;
   return (sqlite3PCachePercentDirty(pPager->pPCache)>=25);
 }
@@ -1997,7 +2001,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
     }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST
       || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL)
     ){
-      rc = zeroJournalHdr(pPager, hasMaster);
+      rc = zeroJournalHdr(pPager, hasMaster||pPager->tempFile);
       pPager->journalOff = 0;
     }else{
       /* This branch may be executed with Pager.journalMode==MEMORY if
@@ -2032,12 +2036,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
   sqlite3BitvecDestroy(pPager->pInJournal);
   pPager->pInJournal = 0;
   pPager->nRec = 0;
-  if( MEMDB || pagerFlushOnCommit(pPager) ){
-    sqlite3PcacheCleanAll(pPager->pPCache);
-  }else{
-    sqlite3PcacheClearWritable(pPager->pPCache);
+  if( rc==SQLITE_OK ){
+    if( pagerFlushOnCommit(pPager, bCommit) ){
+      sqlite3PcacheCleanAll(pPager->pPCache);
+    }else{
+      sqlite3PcacheClearWritable(pPager->pPCache);
+    }
+    sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize);
   }
-  sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize);
 
   if( pagerUseWal(pPager) ){
     /* Drop the WAL write-lock, if any. Also, if the connection was in 
@@ -2371,7 +2377,6 @@ static int pager_playback_one_page(
     assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 );
     pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK;
     if( rc!=SQLITE_OK ) return rc;
-    pPg->flags &= ~PGHDR_NEED_READ;
     sqlite3PcacheMakeDirty(pPg);
   }
   if( pPg ){
@@ -2385,33 +2390,10 @@ static int pager_playback_one_page(
     pData = pPg->pData;
     memcpy(pData, (u8*)aData, pPager->pageSize);
     pPager->xReiniter(pPg);
-    if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){
-      /* If the contents of this page were just restored from the main 
-      ** journal file, then its content must be as they were when the 
-      ** transaction was first opened. In this case we can mark the page
-      ** as clean, since there will be no need to write it out to the
-      ** database.
-      **
-      ** There is one exception to this rule. If the page is being rolled
-      ** back as part of a savepoint (or statement) rollback from an 
-      ** unsynced portion of the main journal file, then it is not safe
-      ** to mark the page as clean. This is because marking the page as
-      ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is
-      ** already in the journal file (recorded in Pager.pInJournal) and
-      ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to
-      ** again within this transaction, it will be marked as dirty but
-      ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially
-      ** be written out into the database file before its journal file
-      ** segment is synced. If a crash occurs during or following this,
-      ** database corruption may ensue.
-      **
-      ** Update: Another exception is for temp files that are not 
-      ** in-memory databases. In this case the page may have been dirty
-      ** at the start of the transaction.
-      */
-      assert( !pagerUseWal(pPager) );
-      if( pPager->tempFile==0 ) sqlite3PcacheMakeClean(pPg);
-    }
+    /* It used to be that sqlite3PcacheMakeClean(pPg) was called here.  But
+    ** that call was dangerous and had no detectable benefit since the cache
+    ** is normally cleaned by sqlite3PcacheCleanAll() after rollback and so
+    ** has been removed. */
     pager_set_pagehash(pPg);
 
     /* If this was page 1, then restore the value of Pager.dbFileVers.
@@ -6022,6 +6004,7 @@ void sqlite3PagerDontWrite(PgHdr *pPg){
     IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno))
     pPg->flags |= PGHDR_DONT_WRITE;
     pPg->flags &= ~PGHDR_WRITEABLE;
+    testcase( pPg->flags & PGHDR_NEED_SYNC );
     pager_set_pagehash(pPg);
   }
 }
@@ -6217,7 +6200,7 @@ int sqlite3PagerCommitPhaseOne(
   if( NEVER(pPager->errCode) ) return pPager->errCode;
 
   /* Provide the ability to easily simulate an I/O error during testing */
-  if( (rc = sqlite3FaultSim(400))!=SQLITE_OK ) return rc;
+  if( sqlite3FaultSim(400) ) return SQLITE_IOERR;
 
   PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", 
       pPager->zFilename, zMaster, pPager->dbSize));
@@ -6227,7 +6210,7 @@ int sqlite3PagerCommitPhaseOne(
 
   assert( MEMDB==0 || pPager->tempFile );
   assert( isOpen(pPager->fd) || pPager->tempFile );
-  if( 0==pagerFlushOnCommit(pPager) ){
+  if( 0==pagerFlushOnCommit(pPager, 1) ){
     /* If this is an in-memory db, or no pages have been written to, or this
     ** function has already been called, it is mostly a no-op.  However, any
     ** backup in progress needs to be restarted.  */
@@ -6852,6 +6835,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){
   /* In order to be able to rollback, an in-memory database must journal
   ** the page we are moving from.
   */
+  assert( pPager->tempFile || !MEMDB );
   if( pPager->tempFile ){
     rc = sqlite3PagerWrite(pPg);
     if( rc ) return rc;
@@ -6926,8 +6910,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){
   ** to exist, in case the transaction needs to roll back.  Use pPgOld
   ** as the original page since it has already been allocated.
   */
-  if( pPager->tempFile ){
-    assert( pPgOld );
+  if( pPager->tempFile && pPgOld ){
     sqlite3PcacheMove(pPgOld, origPgno);
     sqlite3PagerUnrefNotNull(pPgOld);
   }
@@ -7174,16 +7157,6 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){
   return &pPager->pBackup;
 }
 
-#ifndef SQLITE_OMIT_VACUUM
-/*
-** Unless this is an in-memory or temporary database, clear the pager cache.
-*/
-void sqlite3PagerClearCache(Pager *pPager){
-  assert( MEMDB==0 || pPager->tempFile );
-  if( pPager->tempFile==0 ) pager_reset(pPager);
-}
-#endif
-
 #ifndef SQLITE_OMIT_WAL
 /*
 ** This function is called when the user invokes "PRAGMA wal_checkpoint",

src/pager.h

@@ -203,7 +203,6 @@ const char *sqlite3PagerJournalname(Pager*);
 void *sqlite3PagerTempSpace(Pager*);
 int sqlite3PagerIsMemdb(Pager*);
 void sqlite3PagerCacheStat(Pager *, int, int, int *);
-void sqlite3PagerClearCache(Pager *);
 int sqlite3SectorSize(sqlite3_file *);
 
 /* Functions used to truncate the database file. */

src/parse.y

@@ -300,7 +300,8 @@ ccons ::= NULL onconf.
 ccons ::= NOT NULL onconf(R).    {sqlite3AddNotNull(pParse, R);}
 ccons ::= PRIMARY KEY sortorder(Z) onconf(R) autoinc(I).
                                  {sqlite3AddPrimaryKey(pParse,0,R,I,Z);}
-ccons ::= UNIQUE onconf(R).      {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0);}
+ccons ::= UNIQUE onconf(R).      {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0,
+                                   SQLITE_IDXTYPE_UNIQUE);}
 ccons ::= CHECK LP expr(X) RP.   {sqlite3AddCheckConstraint(pParse,X.pExpr);}
 ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R).
                                  {sqlite3CreateForeignKey(pParse,0,&T,TA,R);}
@@ -349,7 +350,8 @@ tcons ::= CONSTRAINT nm(X).      {pParse->constraintName = X;}
 tcons ::= PRIMARY KEY LP sortlist(X) autoinc(I) RP onconf(R).
                                  {sqlite3AddPrimaryKey(pParse,X,R,I,0);}
 tcons ::= UNIQUE LP sortlist(X) RP onconf(R).
-                                 {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0);}
+                                 {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0,
+                                       SQLITE_IDXTYPE_UNIQUE);}
 tcons ::= CHECK LP expr(E) RP onconf.
                                  {sqlite3AddCheckConstraint(pParse,E.pExpr);}
 tcons ::= FOREIGN KEY LP eidlist(FA) RP
@@ -1198,7 +1200,7 @@ cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D)
         ON nm(Y) LP sortlist(Z) RP where_opt(W). {
   sqlite3CreateIndex(pParse, &X, &D, 
                      sqlite3SrcListAppend(pParse->db,0,&Y,0), Z, U,
-                      &S, W, SQLITE_SO_ASC, NE);
+                      &S, W, SQLITE_SO_ASC, NE, SQLITE_IDXTYPE_APPDEF);
 }
 
 %type uniqueflag {int}

src/pcache.c

@@ -14,7 +14,29 @@
 #include "sqliteInt.h"
 
 /*
-** A complete page cache is an instance of this structure.
+** A complete page cache is an instance of this structure.  Every
+** entry in the cache holds a single page of the database file.  The
+** btree layer only operates on the cached copy of the database pages.
+**
+** A page cache entry is "clean" if it exactly matches what is currently
+** on disk.  A page is "dirty" if it has been modified and needs to be
+** persisted to disk.
+**
+** pDirty, pDirtyTail, pSynced:
+**   All dirty pages are linked into the doubly linked list using
+**   PgHdr.pDirtyNext and pDirtyPrev. The list is maintained in LRU order
+**   such that p was added to the list more recently than p->pDirtyNext.
+**   PCache.pDirty points to the first (newest) element in the list and
+**   pDirtyTail to the last (oldest).
+**
+**   The PCache.pSynced variable is used to optimize searching for a dirty
+**   page to eject from the cache mid-transaction. It is better to eject
+**   a page that does not require a journal sync than one that does. 
+**   Therefore, pSynced is maintained to that it *almost* always points
+**   to either the oldest page in the pDirty/pDirtyTail list that has a
+**   clear PGHDR_NEED_SYNC flag or to a page that is older than this one
+**   (so that the right page to eject can be found by following pDirtyPrev
+**   pointers).
 */
 struct PCache {
   PgHdr *pDirty, *pDirtyTail;         /* List of dirty pages in LRU order */
@@ -31,6 +53,95 @@ struct PCache {
   sqlite3_pcache *pCache;             /* Pluggable cache module */
 };
 
+/********************************** Test and Debug Logic **********************/
+/*
+** Debug tracing macros.  Enable by by changing the "0" to "1" and
+** recompiling.
+**
+** When sqlite3PcacheTrace is 1, single line trace messages are issued.
+** When sqlite3PcacheTrace is 2, a dump of the pcache showing all cache entries
+** is displayed for many operations, resulting in a lot of output.
+*/
+#if defined(SQLITE_DEBUG) && 0
+  int sqlite3PcacheTrace = 2;       /* 0: off  1: simple  2: cache dumps */
+  int sqlite3PcacheMxDump = 9999;   /* Max cache entries for pcacheDump() */
+# define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;}
+  void pcacheDump(PCache *pCache){
+    int N;
+    int i, j;
+    sqlite3_pcache_page *pLower;
+    PgHdr *pPg;
+    unsigned char *a;
+  
+    if( sqlite3PcacheTrace<2 ) return;
+    if( pCache->pCache==0 ) return;
+    N = sqlite3PcachePagecount(pCache);
+    if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump;
+    for(i=1; i<=N; i++){
+       pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0);
+       if( pLower==0 ) continue;
+       pPg = (PgHdr*)pLower->pExtra;
+       printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags);
+       a = (unsigned char *)pLower->pBuf;
+       for(j=0; j<12; j++) printf("%02x", a[j]);
+       printf("\n");
+       if( pPg->pPage==0 ){
+         sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0);
+       }
+    }
+  }
+  #else
+# define pcacheTrace(X)
+# define pcacheDump(X)
+#endif
+
+/*
+** Check invariants on a PgHdr entry.  Return true if everything is OK.
+** Return false if any invariant is violated.
+**
+** This routine is for use inside of assert() statements only.  For
+** example:
+**
+**          assert( sqlite3PcachePageSanity(pPg) );
+*/
+#if SQLITE_DEBUG
+int sqlite3PcachePageSanity(PgHdr *pPg){
+  PCache *pCache;
+  assert( pPg!=0 );
+  assert( pPg->pgno>0 );    /* Page number is 1 or more */
+  pCache = pPg->pCache;
+  assert( pCache!=0 );      /* Every page has an associated PCache */
+  if( pPg->flags & PGHDR_CLEAN ){
+    assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */
+    assert( pCache->pDirty!=pPg );          /* CLEAN pages not on dirty list */
+    assert( pCache->pDirtyTail!=pPg );
+  }
+  /* WRITEABLE pages must also be DIRTY */
+  if( pPg->flags & PGHDR_WRITEABLE ){
+    assert( pPg->flags & PGHDR_DIRTY );     /* WRITEABLE implies DIRTY */
+  }
+  /* NEED_SYNC can be set independently of WRITEABLE.  This can happen,
+  ** for example, when using the sqlite3PagerDontWrite() optimization:
+  **    (1)  Page X is journalled, and gets WRITEABLE and NEED_SEEK.
+  **    (2)  Page X moved to freelist, WRITEABLE is cleared
+  **    (3)  Page X reused, WRITEABLE is set again
+  ** If NEED_SYNC had been cleared in step 2, then it would not be reset
+  ** in step 3, and page might be written into the database without first
+  ** syncing the rollback journal, which might cause corruption on a power
+  ** loss.
+  **
+  ** Another example is when the database page size is smaller than the
+  ** disk sector size.  When any page of a sector is journalled, all pages
+  ** in that sector are marked NEED_SYNC even if they are still CLEAN, just
+  ** in case they are later modified, since all pages in the same sector
+  ** must be journalled and synced before any of those pages can be safely
+  ** written.
+  */
+  return 1;
+}
+#endif /* SQLITE_DEBUG */
+
+
 /********************************** Linked List Management ********************/
 
 /* Allowed values for second argument to pcacheManageDirtyList() */
@@ -47,17 +158,16 @@ struct PCache {
 static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){
   PCache *p = pPage->pCache;
 
+  pcacheTrace(("%p.DIRTYLIST.%s %d\n", p,
+                addRemove==1 ? "REMOVE" : addRemove==2 ? "ADD" : "FRONT",
+                pPage->pgno));
   if( addRemove & PCACHE_DIRTYLIST_REMOVE ){
     assert( pPage->pDirtyNext || pPage==p->pDirtyTail );
     assert( pPage->pDirtyPrev || pPage==p->pDirty );
   
     /* Update the PCache1.pSynced variable if necessary. */
     if( p->pSynced==pPage ){
-      PgHdr *pSynced = pPage->pDirtyPrev;
-      while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){
-        pSynced = pSynced->pDirtyPrev;
-      }
-      p->pSynced = pSynced;
+      p->pSynced = pPage->pDirtyPrev;
     }
   
     if( pPage->pDirtyNext ){
@@ -69,10 +179,15 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){
     if( pPage->pDirtyPrev ){
       pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext;
     }else{
+      /* If there are now no dirty pages in the cache, set eCreate to 2. 
+      ** This is an optimization that allows sqlite3PcacheFetch() to skip
+      ** searching for a dirty page to eject from the cache when it might
+      ** otherwise have to.  */
       assert( pPage==p->pDirty );
       p->pDirty = pPage->pDirtyNext;
-      if( p->pDirty==0 && p->bPurgeable ){
-        assert( p->eCreate==1 );
+      assert( p->bPurgeable || p->eCreate==2 );
+      if( p->pDirty==0 ){         /*OPTIMIZATION-IF-TRUE*/
+        assert( p->bPurgeable==0 || p->eCreate==1 );
         p->eCreate = 2;
       }
     }
@@ -94,10 +209,19 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){
       }
     }
     p->pDirty = pPage;
-    if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){
+
+    /* If pSynced is NULL and this page has a clear NEED_SYNC flag, set
+    ** pSynced to point to it. Checking the NEED_SYNC flag is an 
+    ** optimization, as if pSynced points to a page with the NEED_SYNC
+    ** flag set sqlite3PcacheFetchStress() searches through all newer 
+    ** entries of the dirty-list for a page with NEED_SYNC clear anyway.  */
+    if( !p->pSynced 
+     && 0==(pPage->flags&PGHDR_NEED_SYNC)   /*OPTIMIZATION-IF-FALSE*/
+    ){
       p->pSynced = pPage;
     }
   }
+  pcacheDump(p);
 }
 
 /*
@@ -106,7 +230,9 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){
 */
 static void pcacheUnpin(PgHdr *p){
   if( p->pCache->bPurgeable ){
+    pcacheTrace(("%p.UNPIN %d\n", p->pCache, p->pgno));
     sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0);
+    pcacheDump(p->pCache);
   }
 }
 
@@ -176,6 +302,7 @@ int sqlite3PcacheOpen(
   p->pStress = pStress;
   p->szCache = 100;
   p->szSpill = 1;
+  pcacheTrace(("%p.OPEN szPage %d bPurgeable %d\n",p,szPage,bPurgeable));
   return sqlite3PcacheSetPageSize(p, szPage);
 }
 
@@ -198,6 +325,7 @@ int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){
     }
     pCache->pCache = pNew;
     pCache->szPage = szPage;
+    pcacheTrace(("%p.PAGESIZE %d\n",pCache,szPage));
   }
   return SQLITE_OK;
 }
@@ -232,11 +360,13 @@ sqlite3_pcache_page *sqlite3PcacheFetch(
   int createFlag        /* If true, create page if it does not exist already */
 ){
   int eCreate;
+  sqlite3_pcache_page *pRes;
 
   assert( pCache!=0 );
   assert( pCache->pCache!=0 );
   assert( createFlag==3 || createFlag==0 );
   assert( pgno>0 );
+  assert( pCache->eCreate==((pCache->bPurgeable && pCache->pDirty) ? 1 : 2) );
 
   /* eCreate defines what to do if the page does not exist.
   **    0     Do not allocate a new page.  (createFlag==0)
@@ -249,7 +379,10 @@ sqlite3_pcache_page *sqlite3PcacheFetch(
   assert( eCreate==0 || eCreate==1 || eCreate==2 );
   assert( createFlag==0 || pCache->eCreate==eCreate );
   assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) );
-  return sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate);
+  pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate);
+  pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno,
+               createFlag?" create":"",pRes));
+  return pRes;
 }
 
 /*
@@ -276,7 +409,11 @@ int sqlite3PcacheFetchStress(
     ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC
     ** cleared), but if that is not possible settle for any other 
     ** unreferenced dirty page.
-    */
+    **
+    ** If the LRU page in the dirty list that has a clear PGHDR_NEED_SYNC
+    ** flag is currently referenced, then the following may leave pSynced
+    ** set incorrectly (pointing to other than the LRU page with NEED_SYNC
+    ** cleared). This is Ok, as pSynced is just an optimization.  */
     for(pPg=pCache->pSynced; 
         pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); 
         pPg=pPg->pDirtyPrev
@@ -294,7 +431,9 @@ int sqlite3PcacheFetchStress(
                   sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache),
                 numberOfCachePages(pCache));
 #endif
+      pcacheTrace(("%p.SPILL %d\n",pCache,pPg->pgno));
       rc = pCache->xStress(pCache->pStress, pPg);
+      pcacheDump(pCache);
       if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){
         return rc;
       }
@@ -354,6 +493,7 @@ PgHdr *sqlite3PcacheFetchFinish(
   }
   pCache->nRefSum++;
   pPgHdr->nRef++;
+  assert( sqlite3PcachePageSanity(pPgHdr) );
   return pPgHdr;
 }
 
@@ -367,8 +507,11 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){
   if( (--p->nRef)==0 ){
     if( p->flags&PGHDR_CLEAN ){
       pcacheUnpin(p);
-    }else if( p->pDirtyPrev!=0 ){
-      /* Move the page to the head of the dirty list. */
+    }else if( p->pDirtyPrev!=0 ){ /*OPTIMIZATION-IF-FALSE*/
+      /* Move the page to the head of the dirty list. If p->pDirtyPrev==0,
+      ** then page p is already at the head of the dirty list and the
+      ** following call would be a no-op. Hence the OPTIMIZATION-IF-FALSE
+      ** tag above.  */
       pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT);
     }
   }
@@ -379,6 +522,7 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){
 */
 void sqlite3PcacheRef(PgHdr *p){
   assert(p->nRef>0);
+  assert( sqlite3PcachePageSanity(p) );
   p->nRef++;
   p->pCache->nRefSum++;
 }
@@ -390,6 +534,7 @@ void sqlite3PcacheRef(PgHdr *p){
 */
 void sqlite3PcacheDrop(PgHdr *p){
   assert( p->nRef==1 );
+  assert( sqlite3PcachePageSanity(p) );
   if( p->flags&PGHDR_DIRTY ){
     pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE);
   }
@@ -403,13 +548,16 @@ void sqlite3PcacheDrop(PgHdr *p){
 */
 void sqlite3PcacheMakeDirty(PgHdr *p){
   assert( p->nRef>0 );
-  if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){
+  assert( sqlite3PcachePageSanity(p) );
+  if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){    /*OPTIMIZATION-IF-FALSE*/
     p->flags &= ~PGHDR_DONT_WRITE;
     if( p->flags & PGHDR_CLEAN ){
       p->flags ^= (PGHDR_DIRTY|PGHDR_CLEAN);
+      pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno));
       assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY );
       pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD);
     }
+    assert( sqlite3PcachePageSanity(p) );
   }
 }
 
@@ -418,11 +566,14 @@ void sqlite3PcacheMakeDirty(PgHdr *p){
 ** make it so.
 */
 void sqlite3PcacheMakeClean(PgHdr *p){
-  if( (p->flags & PGHDR_DIRTY) ){
+  assert( sqlite3PcachePageSanity(p) );
+  if( ALWAYS((p->flags & PGHDR_DIRTY)!=0) ){
     assert( (p->flags & PGHDR_CLEAN)==0 );
     pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE);
     p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE);
     p->flags |= PGHDR_CLEAN;
+    pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno));
+    assert( sqlite3PcachePageSanity(p) );
     if( p->nRef==0 ){
       pcacheUnpin(p);
     }
@@ -434,6 +585,7 @@ void sqlite3PcacheMakeClean(PgHdr *p){
 */
 void sqlite3PcacheCleanAll(PCache *pCache){
   PgHdr *p;
+  pcacheTrace(("%p.CLEAN-ALL\n",pCache));
   while( (p = pCache->pDirty)!=0 ){
     sqlite3PcacheMakeClean(p);
   }
@@ -444,6 +596,7 @@ void sqlite3PcacheCleanAll(PCache *pCache){
 */
 void sqlite3PcacheClearWritable(PCache *pCache){
   PgHdr *p;
+  pcacheTrace(("%p.CLEAR-WRITEABLE\n",pCache));
   for(p=pCache->pDirty; p; p=p->pDirtyNext){
     p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE);
   }
@@ -468,6 +621,8 @@ void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){
   PCache *pCache = p->pCache;
   assert( p->nRef>0 );
   assert( newPgno>0 );
+  assert( sqlite3PcachePageSanity(p) );
+  pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno));
   sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno);
   p->pgno = newPgno;
   if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){
@@ -488,6 +643,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){
   if( pCache->pCache ){
     PgHdr *p;
     PgHdr *pNext;
+    pcacheTrace(("%p.TRUNCATE %d\n",pCache,pgno));
     for(p=pCache->pDirty; p; p=pNext){
       pNext = p->pDirtyNext;
       /* This routine never gets call with a positive pgno except right
@@ -518,6 +674,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){
 */
 void sqlite3PcacheClose(PCache *pCache){
   assert( pCache->pCache!=0 );
+  pcacheTrace(("%p.CLOSE\n",pCache));
   sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache);
 }
 
@@ -530,29 +687,31 @@ void sqlite3PcacheClear(PCache *pCache){
 
 /*
 ** Merge two lists of pages connected by pDirty and in pgno order.
-** Do not both fixing the pDirtyPrev pointers.
+** Do not bother fixing the pDirtyPrev pointers.
 */
 static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){
   PgHdr result, *pTail;
   pTail = &result;
-  while( pA && pB ){
+  assert( pA!=0 && pB!=0 );
+  for(;;){
     if( pA->pgno<pB->pgno ){
       pTail->pDirty = pA;
       pTail = pA;
       pA = pA->pDirty;
+      if( pA==0 ){
+        pTail->pDirty = pB;
+        break;
+      }
     }else{
       pTail->pDirty = pB;
       pTail = pB;
       pB = pB->pDirty;
+      if( pB==0 ){
+        pTail->pDirty = pA;
+        break;
+      }
     }
   }
-  if( pA ){
-    pTail->pDirty = pA;
-  }else if( pB ){
-    pTail->pDirty = pB;
-  }else{
-    pTail->pDirty = 0;
-  }
   return result.pDirty;
 }
 
@@ -593,7 +752,8 @@ static PgHdr *pcacheSortDirtyList(PgHdr *pIn){
   }
   p = a[0];
   for(i=1; i<N_SORT_BUCKET; i++){
-    p = pcacheMergeDirtyList(p, a[i]);
+    if( a[i]==0 ) continue;
+    p = p ? pcacheMergeDirtyList(p, a[i]) : a[i];
   }
   return p;
 }

src/pcache.h

@@ -26,7 +26,7 @@ struct PgHdr {
   sqlite3_pcache_page *pPage;    /* Pcache object page handle */
   void *pData;                   /* Page data */
   void *pExtra;                  /* Extra content */
-  PgHdr *pDirty;                 /* Transient list of dirty pages */
+  PgHdr *pDirty;                 /* Transient list of dirty sorted by pgno */
   Pager *pPager;                 /* The pager this page is part of */
   Pgno pgno;                     /* Page number for this page */
 #ifdef SQLITE_CHECK_PAGES
@@ -51,11 +51,10 @@ struct PgHdr {
 #define PGHDR_WRITEABLE       0x004  /* Journaled and ready to modify */
 #define PGHDR_NEED_SYNC       0x008  /* Fsync the rollback journal before
                                      ** writing this page to the database */
-#define PGHDR_NEED_READ       0x010  /* Content is unread */
-#define PGHDR_DONT_WRITE      0x020  /* Do not write content to disk */
-#define PGHDR_MMAP            0x040  /* This is an mmap page object */
+#define PGHDR_DONT_WRITE      0x010  /* Do not write content to disk */
+#define PGHDR_MMAP            0x020  /* This is an mmap page object */
 
-#define PGHDR_WAL_APPEND      0x080  /* Appended to wal file */
+#define PGHDR_WAL_APPEND      0x040  /* Appended to wal file */
 
 /* Initialize and shutdown the page cache subsystem */
 int sqlite3PcacheInitialize(void);
@@ -138,6 +137,11 @@ int sqlite3PcachePagecount(PCache*);
 void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *));
 #endif
 
+#if defined(SQLITE_DEBUG)
+/* Check invariants on a PgHdr object */
+int sqlite3PcachePageSanity(PgHdr*);
+#endif
+
 /* Set and get the suggested cache-size for the specified pager-cache.
 **
 ** If no global maximum is configured, then the system attempts to limit

src/pragma.c

@@ -1030,7 +1030,7 @@ void sqlite3Pragma(
       ** compiler (eg. count_changes). So add an opcode to expire all
       ** compiled SQL statements after modifying a pragma value.
       */
-      sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
+      sqlite3VdbeAddOp0(v, OP_Expire);
       setAllPagerFlags(db);
     }
     break;
@@ -1052,7 +1052,7 @@ void sqlite3Pragma(
   */
   case PragTyp_TABLE_INFO: if( zRight ){
     Table *pTab;
-    pTab = sqlite3FindTable(db, zRight, zDb);
+    pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb);
     if( pTab ){
       static const char *azCol[] = {
          "cid", "name", "type", "notnull", "dflt_value", "pk"
@@ -1334,12 +1334,10 @@ void sqlite3Pragma(
             sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow);
             sqlite3ColumnDefault(v, pTab, iKey, regRow);
             sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v);
-            sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, 
-               sqlite3VdbeCurrentAddr(v)+3); VdbeCoverage(v);
           }else{
             sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow);
           }
-          sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v);
+          sqlite3VdbeAddOp3(v, OP_SeekRowid, i, 0, regRow); VdbeCoverage(v);
           sqlite3VdbeGoto(v, addrOk);
           sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2);
         }else{

src/rowset.c

@@ -242,28 +242,26 @@ static struct RowSetEntry *rowSetEntryMerge(
   struct RowSetEntry *pTail;
 
   pTail = &head;
-  while( pA && pB ){
+  assert( pA!=0 && pB!=0 );
+  for(;;){
     assert( pA->pRight==0 || pA->v<=pA->pRight->v );
     assert( pB->pRight==0 || pB->v<=pB->pRight->v );
-    if( pA->v<pB->v ){
-      pTail->pRight = pA;
+    if( pA->v<=pB->v ){
+      if( pA->v<pB->v ) pTail = pTail->pRight = pA;
       pA = pA->pRight;
-      pTail = pTail->pRight;
-    }else if( pB->v<pA->v ){
-      pTail->pRight = pB;
-      pB = pB->pRight;
-      pTail = pTail->pRight;
+      if( pA==0 ){
+        pTail->pRight = pB;
+        break;
+      }
     }else{
-      pA = pA->pRight;
+      pTail = pTail->pRight = pB;
+      pB = pB->pRight;
+      if( pB==0 ){
+        pTail->pRight = pA;
+        break;
+      }
     }
   }
-  if( pA ){
-    assert( pA->pRight==0 || pA->v<=pA->pRight->v );
-    pTail->pRight = pA;
-  }else{
-    assert( pB==0 || pB->pRight==0 || pB->v<=pB->pRight->v );
-    pTail->pRight = pB;
-  }
   return head.pRight;
 }
 
@@ -286,9 +284,10 @@ static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){
     aBucket[i] = pIn;
     pIn = pNext;
   }
-  pIn = 0;
-  for(i=0; i<sizeof(aBucket)/sizeof(aBucket[0]); i++){
-    pIn = rowSetEntryMerge(pIn, aBucket[i]);
+  pIn = aBucket[0];
+  for(i=1; i<sizeof(aBucket)/sizeof(aBucket[0]); i++){
+    if( aBucket[i]==0 ) continue;
+    pIn = pIn ? rowSetEntryMerge(pIn, aBucket[i]) : aBucket[i];
   }
   return pIn;
 }

src/select.c

@@ -56,6 +56,7 @@ struct SortCtx {
   int addrSortIndex;    /* Address of the OP_SorterOpen or OP_OpenEphemeral */
   int labelDone;        /* Jump here when done, ex: LIMIT reached */
   u8 sortFlags;         /* Zero or more SORTFLAG_* bits */
+  u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */
 };
 #define SORTFLAG_UseSorter  0x01   /* Use SorterOpen instead of OpenEphemeral */
 
@@ -589,9 +590,30 @@ static void pushOntoSorter(
   sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord);
   if( iLimit ){
     int addr;
+    int r1 = 0;
+    /* Fill the sorter until it contains LIMIT+OFFSET entries.  (The iLimit
+    ** register is initialized with value of LIMIT+OFFSET.)  After the sorter
+    ** fills up, delete the least entry in the sorter after each insert.
+    ** Thus we never hold more than the LIMIT+OFFSET rows in memory at once */
     addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v);
     sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor);
+    if( pSort->bOrderedInnerLoop ){
+      r1 = ++pParse->nMem;
+      sqlite3VdbeAddOp3(v, OP_Column, pSort->iECursor, nExpr, r1);
+      VdbeComment((v, "seq"));
+    }
     sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor);
+    if( pSort->bOrderedInnerLoop ){
+      /* If the inner loop is driven by an index such that values from
+      ** the same iteration of the inner loop are in sorted order, then
+      ** immediately jump to the next iteration of an inner loop if the
+      ** entry from the current iteration does not fit into the top
+      ** LIMIT+OFFSET entries of the sorter. */
+      int iBrk = sqlite3VdbeCurrentAddr(v) + 2;
+      sqlite3VdbeAddOp3(v, OP_Eq, regBase+nExpr, iBrk, r1);
+      sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
+      VdbeCoverage(v);
+    }
     sqlite3VdbeJumpHere(v, addr);
   }
 }
@@ -1006,7 +1028,7 @@ static void selectInnerLoop(
 */
 KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){
   int nExtra = (N+X)*(sizeof(CollSeq*)+1);
-  KeyInfo *p = sqlite3Malloc(sizeof(KeyInfo) + nExtra);
+  KeyInfo *p = sqlite3DbMallocRaw(db, sizeof(KeyInfo) + nExtra);
   if( p ){
     p->aSortOrder = (u8*)&p->aColl[N+X];
     p->nField = (u16)N;
@@ -1028,7 +1050,7 @@ void sqlite3KeyInfoUnref(KeyInfo *p){
   if( p ){
     assert( p->nRef>0 );
     p->nRef--;
-    if( p->nRef==0 ) sqlite3DbFree(0, p);
+    if( p->nRef==0 ) sqlite3DbFree(p->db, p);
   }
 }
 
@@ -5176,6 +5198,7 @@ int sqlite3Select(
     }
     if( sSort.pOrderBy ){
       sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo);
+      sSort.bOrderedInnerLoop = sqlite3WhereOrderedInnerLoop(pWInfo);
       if( sSort.nOBSat==sSort.pOrderBy->nExpr ){
         sSort.pOrderBy = 0;
       }

src/sqlite.h.in

@@ -506,6 +506,7 @@ int sqlite3_exec(
 #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8))
 #define SQLITE_WARNING_AUTOINDEX       (SQLITE_WARNING | (1<<8))
 #define SQLITE_AUTH_USER               (SQLITE_AUTH | (1<<8))
+#define SQLITE_OK_LOAD_PERMANENTLY     (SQLITE_OK | (1<<8))
 
 /*
 ** CAPI3REF: Flags For File Open Operations

src/sqliteInt.h

@@ -411,7 +411,7 @@
 ** be true and false so that the unreachable code they specify will
 ** not be counted as untested code.
 */
-#if defined(SQLITE_COVERAGE_TEST)
+#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
 # define ALWAYS(X)      (1)
 # define NEVER(X)       (0)
 #elif !defined(NDEBUG)
@@ -2485,7 +2485,7 @@ struct SrcList {
     int regReturn;    /* Register holding return address of addrFillSub */
     int regResult;    /* Registers holding results of a co-routine */
     struct {
-      u8 jointype;      /* Type of join between this able and the previous */
+      u8 jointype;      /* Type of join between this table and the previous */
       unsigned notIndexed :1;    /* True if there is a NOT INDEXED clause */
       unsigned isIndexedBy :1;   /* True if there is an INDEXED BY clause */
       unsigned isTabFunc :1;     /* True if table-valued-function syntax */
@@ -2531,19 +2531,20 @@ struct SrcList {
 #define WHERE_ORDERBY_MIN      0x0001 /* ORDER BY processing for min() func */
 #define WHERE_ORDERBY_MAX      0x0002 /* ORDER BY processing for max() func */
 #define WHERE_ONEPASS_DESIRED  0x0004 /* Want to do one-pass UPDATE/DELETE */
-#define WHERE_DUPLICATES_OK    0x0008 /* Ok to return a row more than once */
-#define WHERE_OMIT_OPEN_CLOSE  0x0010 /* Table cursors are already open */
-#define WHERE_FORCE_TABLE      0x0020 /* Do not use an index-only search */
-#define WHERE_ONETABLE_ONLY    0x0040 /* Only code the 1st table in pTabList */
-#define WHERE_NO_AUTOINDEX     0x0080 /* Disallow automatic indexes */
-#define WHERE_GROUPBY          0x0100 /* pOrderBy is really a GROUP BY */
-#define WHERE_DISTINCTBY       0x0200 /* pOrderby is really a DISTINCT clause */
-#define WHERE_WANT_DISTINCT    0x0400 /* All output needs to be distinct */
-#define WHERE_SORTBYGROUP      0x0800 /* Support sqlite3WhereIsSorted() */
-                           /*  0x1000    not currently used */
-#define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */
-#define WHERE_USE_LIMIT        0x4000 /* There is a constant LIMIT clause */
-#define WHERE_SEEK_TABLE       0x8000 /* Do not defer seeks on main table */
+#define WHERE_ONEPASS_MULTIROW 0x0008 /* ONEPASS is ok with multiple rows */
+#define WHERE_DUPLICATES_OK    0x0010 /* Ok to return a row more than once */
+#define WHERE_OR_SUBCLAUSE     0x0020 /* Processing a sub-WHERE as part of
+                                      ** the OR optimization  */
+#define WHERE_GROUPBY          0x0040 /* pOrderBy is really a GROUP BY */
+#define WHERE_DISTINCTBY       0x0080 /* pOrderby is really a DISTINCT clause */
+#define WHERE_WANT_DISTINCT    0x0100 /* All output needs to be distinct */
+#define WHERE_SORTBYGROUP      0x0200 /* Support sqlite3WhereIsSorted() */
+#define WHERE_SEEK_TABLE       0x0400 /* Do not defer seeks on main table */
+#define WHERE_ORDERBY_LIMIT    0x0800 /* ORDERBY+LIMIT on the inner loop */
+                        /*     0x1000    not currently used */
+                        /*     0x2000    not currently used */
+#define WHERE_USE_LIMIT        0x4000 /* Use the LIMIT in cost estimates */
+                        /*     0x8000    not currently used */
 
 /* Allowed return values from sqlite3WhereIsDistinct()
 */
@@ -3616,8 +3617,8 @@ void sqlite3SrcListAssignCursors(Parse*, SrcList*);
 void sqlite3IdListDelete(sqlite3*, IdList*);
 void sqlite3SrcListDelete(sqlite3*, SrcList*);
 Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**);
-Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
-                          Expr*, int, int);
+void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
+                          Expr*, int, int, u8);
 void sqlite3DropIndex(Parse*, SrcList*, int);
 int sqlite3Select(Parse*, Select*, SelectDest*);
 Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*,
@@ -3636,6 +3637,7 @@ void sqlite3WhereEnd(WhereInfo*);
 LogEst sqlite3WhereOutputRowCount(WhereInfo*);
 int sqlite3WhereIsDistinct(WhereInfo*);
 int sqlite3WhereIsOrdered(WhereInfo*);
+int sqlite3WhereOrderedInnerLoop(WhereInfo*);
 int sqlite3WhereIsSorted(WhereInfo*);
 int sqlite3WhereContinueLabel(WhereInfo*);
 int sqlite3WhereBreakLabel(WhereInfo*);
@@ -3669,8 +3671,10 @@ void sqlite3ExprIfTrue(Parse*, Expr*, int, int);
 void sqlite3ExprIfFalse(Parse*, Expr*, int, int);
 void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int);
 Table *sqlite3FindTable(sqlite3*,const char*, const char*);
-Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*);
-Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *);
+#define LOCATE_VIEW    0x01
+#define LOCATE_NOERR   0x02
+Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
+Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
 Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
 void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
 void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);

src/tclsqlite.c

@@ -2325,6 +2325,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   */
   case DB_EXISTS: 
   case DB_ONECOLUMN: {
+    Tcl_Obj *pResult = 0;
     DbEvalContext sEval;
     if( objc!=3 ){
       Tcl_WrongNumArgs(interp, 2, objv, "SQL");
@@ -2335,14 +2336,15 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     rc = dbEvalStep(&sEval);
     if( choice==DB_ONECOLUMN ){
       if( rc==TCL_OK ){
-        Tcl_SetObjResult(interp, dbEvalColumnValue(&sEval, 0));
+        pResult = dbEvalColumnValue(&sEval, 0);
       }else if( rc==TCL_BREAK ){
         Tcl_ResetResult(interp);
       }
     }else if( rc==TCL_BREAK || rc==TCL_OK ){
-      Tcl_SetObjResult(interp, Tcl_NewBooleanObj(rc==TCL_OK));
+      pResult = Tcl_NewBooleanObj(rc==TCL_OK);
     }
     dbEvalFinalize(&sEval);
+    if( pResult ) Tcl_SetObjResult(interp, pResult);
 
     if( rc==TCL_BREAK ){
       rc = TCL_OK;

src/test1.c

@@ -6584,6 +6584,7 @@ static int tclLoadStaticExtensionCmd(
 ){
   extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_csv_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_eval_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_fileio_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*);
@@ -6601,6 +6602,7 @@ static int tclLoadStaticExtensionCmd(
   } aExtension[] = {
     { "amatch",                sqlite3_amatch_init               },
     { "closure",               sqlite3_closure_init              },
+    { "csv",                   sqlite3_csv_init                  },
     { "eval",                  sqlite3_eval_init                 },
     { "fileio",                sqlite3_fileio_init               },
     { "fuzzer",                sqlite3_fuzzer_init               },

src/test3.c

@@ -385,8 +385,7 @@ static int btree_payload_size(
   const char **argv      /* Text of each argument */
 ){
   BtCursor *pCur;
-  int n2;
-  u64 n1;
+  u32 n;
   char zBuf[50];
 
   if( argc!=2 ){
@@ -396,17 +395,9 @@ static int btree_payload_size(
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
   sqlite3BtreeEnter(pCur->pBtree);
-
-  /* The cursor may be in "require-seek" state. If this is the case, the
-  ** call to BtreeDataSize() will fix it. */
-  sqlite3BtreeDataSize(pCur, (u32*)&n2);
-  if( pCur->apPage[pCur->iPage]->intKey ){
-    n1 = 0;
-  }else{
-    sqlite3BtreeKeySize(pCur, (i64*)&n1);
-  }
+  n = sqlite3BtreePayloadSize(pCur);
   sqlite3BtreeLeave(pCur->pBtree);
-  sqlite3_snprintf(sizeof(zBuf),zBuf, "%d", (int)(n1+n2));
+  sqlite3_snprintf(sizeof(zBuf),zBuf, "%u", n);
   Tcl_AppendResult(interp, zBuf, 0);
   return SQLITE_OK;
 }
@@ -618,27 +609,27 @@ static int btree_insert(
 ){
   BtCursor *pCur;
   int rc;
-  void *pKey = 0;
-  int nKey = 0;
-  void *pData = 0;
-  int nData = 0;
+  BtreePayload x;
 
   if( objc!=4 && objc!=3 ){
     Tcl_WrongNumArgs(interp, 1, objv, "?-intkey? CSR KEY VALUE");
     return TCL_ERROR;
   }
 
+  memset(&x, 0, sizeof(x));
   if( objc==4 ){
-    if( Tcl_GetIntFromObj(interp, objv[2], &nKey) ) return TCL_ERROR;
-    pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &nData);
+    if( Tcl_GetIntFromObj(interp, objv[2], &rc) ) return TCL_ERROR;
+    x.nKey = rc;
+    x.pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &x.nData);
   }else{
-    pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &nKey);
+    x.pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &rc);
+    x.nKey = rc;
   }
   pCur = (BtCursor*)sqlite3TestTextToPtr(Tcl_GetString(objv[1]));
 
   sqlite3_mutex_enter(pCur->pBtree->db->mutex);
   sqlite3BtreeEnter(pCur->pBtree);
-  rc = sqlite3BtreeInsert(pCur, pKey, nKey, pData, nData, 0, 0, 0);
+  rc = sqlite3BtreeInsert(pCur, &x, 0, 0);
   sqlite3BtreeLeave(pCur->pBtree);
   sqlite3_mutex_leave(pCur->pBtree->db->mutex);
 

src/test6.c

@@ -215,7 +215,9 @@ static int writeListSync(CrashFile *pFile, int isCrash){
   }
 
 #ifdef TRACE_CRASHTEST
-  printf("Sync %s (is %s crash)\n", pFile->zName, (isCrash?"a":"not a"));
+  if( pFile ){
+    printf("Sync %s (is %s crash)\n", pFile->zName, (isCrash?"a":"not a"));
+  }
 #endif
 
   ppPtr = &g.pWriteList;
@@ -799,6 +801,27 @@ static int processDevSymArgs(
   return TCL_OK;
 }
 
+/*
+** tclcmd:   sqlite3_crash_now
+**
+** Simulate a crash immediately. This function does not return 
+** (writeListSync() calls exit(-1)).
+*/
+static int crashNowCmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }
+  writeListSync(0, 1);
+  assert( 0 );
+  return TCL_OK;
+}
+
 /*
 ** tclcmd:   sqlite_crash_enable ENABLE
 **
@@ -1034,6 +1057,7 @@ int Sqlitetest6_Init(Tcl_Interp *interp){
 #ifndef SQLITE_OMIT_DISKIO
   Tcl_CreateObjCommand(interp, "sqlite3_crash_enable", crashEnableCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "sqlite3_crashparams", crashParamsObjCmd, 0, 0);
+  Tcl_CreateObjCommand(interp, "sqlite3_crash_now", crashNowCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "sqlite3_simulate_device", devSymObjCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "unregister_devsim", dsUnregisterObjCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "register_jt_vfs", jtObjCmd, 0, 0);

src/treeview.c

@@ -445,6 +445,12 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
       break;
     }
 #endif
+    case TK_MATCH: {
+      sqlite3TreeViewLine(pView, "MATCH {%d:%d}%s",
+                          pExpr->iTable, pExpr->iColumn, zFlgs);
+      sqlite3TreeViewExpr(pView, pExpr->pRight, 0);
+      break;
+    }
     default: {
       sqlite3TreeViewLine(pView, "op=%d", pExpr->op);
       break;

src/vdbe.c

@@ -2016,6 +2016,7 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
       if( (flags1 | flags3)&MEM_Str ){
         if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
           applyNumericAffinity(pIn1,0);
+          flags3 = pIn3->flags;
         }
         if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
           applyNumericAffinity(pIn3,0);
@@ -2028,6 +2029,7 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
         sqlite3VdbeMemStringify(pIn1, encoding, 1);
         testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) );
         flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask);
+        flags3 = pIn3->flags;
       }
       if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){
         testcase( pIn3->flags & MEM_Int );
@@ -2380,7 +2382,6 @@ case OP_NotNull: {            /* same as TK_NOTNULL, jump, in1 */
 ** skipped for length() and all content loading can be skipped for typeof().
 */
 case OP_Column: {
-  i64 payloadSize64; /* Number of bytes in the record */
   int p2;            /* column number to retrieve */
   VdbeCursor *pC;    /* The VDBE cursor */
   BtCursor *pCrsr;   /* The BTree cursor */
@@ -2403,6 +2404,7 @@ case OP_Column: {
 
   /* If the cursor cache is stale, bring it up-to-date */
   rc = sqlite3VdbeCursorMoveto(&pC, &p2);
+  if( rc ) goto abort_due_to_error;
 
   assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   pDest = &aMem[pOp->p3];
@@ -2416,8 +2418,7 @@ case OP_Column: {
   assert( pC->eCurType!=CURTYPE_SORTER );
   pCrsr = pC->uc.pCursor;
 
-  if( rc ) goto abort_due_to_error;
-  if( pC->cacheStatus!=p->cacheCtr ){
+  if( pC->cacheStatus!=p->cacheCtr ){                /*OPTIMIZATION-IF-FALSE*/
     if( pC->nullRow ){
       if( pC->eCurType==CURTYPE_PSEUDO ){
         assert( pC->uc.pseudoTableReg>0 );
@@ -2433,22 +2434,9 @@ case OP_Column: {
     }else{
       assert( pC->eCurType==CURTYPE_BTREE );
       assert( pCrsr );
-      if( pC->isTable==0 ){
-        assert( sqlite3BtreeCursorIsValid(pCrsr) );
-        VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64);
-        assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */
-        /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the
-        ** payload size, so it is impossible for payloadSize64 to be
-        ** larger than 32 bits. */
-        assert( (payloadSize64 & SQLITE_MAX_U32)==(u64)payloadSize64 );
-        pC->aRow = sqlite3BtreeKeyFetch(pCrsr, &avail);
-        pC->payloadSize = (u32)payloadSize64;
-      }else{
-        assert( sqlite3BtreeCursorIsValid(pCrsr) );
-        VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &pC->payloadSize);
-        assert( rc==SQLITE_OK );   /* DataSize() cannot fail */
-        pC->aRow = sqlite3BtreeDataFetch(pCrsr, &avail);
-      }
+      assert( sqlite3BtreeCursorIsValid(pCrsr) );
+      pC->payloadSize = sqlite3BtreePayloadSize(pCrsr);
+      pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &avail);
       assert( avail<=65536 );  /* Maximum page size is 64KiB */
       if( pC->payloadSize <= (u32)avail ){
         pC->szRow = pC->payloadSize;
@@ -2464,7 +2452,7 @@ case OP_Column: {
     aOffset[0] = offset;
 
 
-    if( avail<offset ){
+    if( avail<offset ){      /*OPTIMIZATION-IF-FALSE*/
       /* pC->aRow does not have to hold the entire row, but it does at least
       ** need to cover the header of the record.  If pC->aRow does not contain
       ** the complete header, then set it to zero, forcing the header to be
@@ -2485,14 +2473,15 @@ case OP_Column: {
         rc = SQLITE_CORRUPT_BKPT;
         goto abort_due_to_error;
       }
+    }else if( offset>0 ){ /*OPTIMIZATION-IF-TRUE*/
+      /* The following goto is an optimization.  It can be omitted and
+      ** everything will still work.  But OP_Column is measurably faster
+      ** by skipping the subsequent conditional, which is always true.
+      */
+      zData = pC->aRow;
+      assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
+      goto op_column_read_header;
     }
-
-    /* The following goto is an optimization.  It can be omitted and
-    ** everything will still work.  But OP_Column is measurably faster
-    ** by skipping the subsequent conditional, which is always true.
-    */
-    assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
-    goto op_column_read_header;
   }
 
   /* Make sure at least the first p2+1 entries of the header have been
@@ -2502,7 +2491,6 @@ case OP_Column: {
     /* If there is more header available for parsing in the record, try
     ** to extract additional fields up through the p2+1-th field 
     */
-    op_column_read_header:
     if( pC->iHdrOffset<aOffset[0] ){
       /* Make sure zData points to enough of the record to cover the header. */
       if( pC->aRow==0 ){
@@ -2515,11 +2503,11 @@ case OP_Column: {
       }
   
       /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */
+    op_column_read_header:
       i = pC->nHdrParsed;
       offset64 = aOffset[i];
       zHdr = zData + pC->iHdrOffset;
       zEndHdr = zData + aOffset[0];
-      assert( i<=p2 && zHdr<zEndHdr );
       do{
         if( (t = zHdr[0])<0x80 ){
           zHdr++;
@@ -2531,9 +2519,7 @@ case OP_Column: {
         pC->aType[i++] = t;
         aOffset[i] = (u32)(offset64 & 0xffffffff);
       }while( i<=p2 && zHdr<zEndHdr );
-      pC->nHdrParsed = i;
-      pC->iHdrOffset = (u32)(zHdr - zData);
-  
+
       /* The record is corrupt if any of the following are true:
       ** (1) the bytes of the header extend past the declared header size
       ** (2) the entire header was used but not all data was used
@@ -2546,8 +2532,10 @@ case OP_Column: {
         rc = SQLITE_CORRUPT_BKPT;
         goto abort_due_to_error;
       }
-      if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
 
+      pC->nHdrParsed = i;
+      pC->iHdrOffset = (u32)(zHdr - zData);
+      if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
     }else{
       t = 0;
     }
@@ -2575,9 +2563,10 @@ case OP_Column: {
   assert( p2<pC->nHdrParsed );
   assert( rc==SQLITE_OK );
   assert( sqlite3VdbeCheckMemInvariants(pDest) );
-  if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest);
+  if( VdbeMemDynamic(pDest) ){
+    sqlite3VdbeMemSetNull(pDest);
+  }
   assert( t==pC->aType[p2] );
-  pDest->enc = encoding;
   if( pC->szRow>=aOffset[p2+1] ){
     /* This is the common case where the desired content fits on the original
     ** page - where the content is not on an overflow page */
@@ -2591,6 +2580,7 @@ case OP_Column: {
       */
       static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term };
       pDest->n = len = (t-12)/2;
+      pDest->enc = encoding;
       if( pDest->szMalloc < len+2 ){
         pDest->flags = MEM_Null;
         if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem;
@@ -2603,6 +2593,7 @@ case OP_Column: {
       pDest->flags = aFlag[t&1];
     }
   }else{
+    pDest->enc = encoding;
     /* This branch happens only when content is on overflow pages */
     if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0
           && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0))
@@ -4020,6 +4011,30 @@ case OP_Found: {        /* jump, in3 */
   break;
 }
 
+/* Opcode: SeekRowid P1 P2 P3 * *
+** Synopsis: intkey=r[P3]
+**
+** P1 is the index of a cursor open on an SQL table btree (with integer
+** keys).  If register P3 does not contain an integer or if P1 does not
+** contain a record with rowid P3 then jump immediately to P2.  
+** Or, if P2 is 0, raise an SQLITE_CORRUPT error. If P1 does contain
+** a record with rowid P3 then 
+** leave the cursor pointing at that record and fall through to the next
+** instruction.
+**
+** The OP_NotExists opcode performs the same operation, but with OP_NotExists
+** the P3 register must be guaranteed to contain an integer value.  With this
+** opcode, register P3 might not contain an integer.
+**
+** The OP_NotFound opcode performs the same operation on index btrees
+** (with arbitrary multi-value keys).
+**
+** This opcode leaves the cursor in a state where it cannot be advanced
+** in either direction.  In other words, the Next and Prev opcodes will
+** not work following this opcode.
+**
+** See also: Found, NotFound, NoConflict, SeekRowid
+*/
 /* Opcode: NotExists P1 P2 P3 * *
 ** Synopsis: intkey=r[P3]
 **
@@ -4030,6 +4045,10 @@ case OP_Found: {        /* jump, in3 */
 ** leave the cursor pointing at that record and fall through to the next
 ** instruction.
 **
+** The OP_SeekRowid opcode performs the same operation but also allows the
+** P3 register to contain a non-integer value, in which case the jump is
+** always taken.  This opcode requires that P3 always contain an integer.
+**
 ** The OP_NotFound opcode performs the same operation on index btrees
 ** (with arbitrary multi-value keys).
 **
@@ -4037,14 +4056,21 @@ case OP_Found: {        /* jump, in3 */
 ** in either direction.  In other words, the Next and Prev opcodes will
 ** not work following this opcode.
 **
-** See also: Found, NotFound, NoConflict
+** See also: Found, NotFound, NoConflict, SeekRowid
 */
-case OP_NotExists: {        /* jump, in3 */
+case OP_SeekRowid: {        /* jump, in3 */
   VdbeCursor *pC;
   BtCursor *pCrsr;
   int res;
   u64 iKey;
 
+  pIn3 = &aMem[pOp->p3];
+  if( (pIn3->flags & MEM_Int)==0 ){
+    applyAffinity(pIn3, SQLITE_AFF_NUMERIC, encoding);
+    if( (pIn3->flags & MEM_Int)==0 ) goto jump_to_p2;
+  }
+  /* Fall through into OP_NotExists */
+case OP_NotExists:          /* jump, in3 */
   pIn3 = &aMem[pOp->p3];
   assert( pIn3->flags & MEM_Int );
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
@@ -4163,8 +4189,7 @@ case OP_NewRowid: {           /* out2 */
         v = 1;   /* IMP: R-61914-48074 */
       }else{
         assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) );
-        rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
-        assert( rc==SQLITE_OK );   /* Cannot fail following BtreeLast() */
+        v = sqlite3BtreeIntegerKey(pC->uc.pCursor);
         if( v>=MAX_ROWID ){
           pC->useRandomRowid = 1;
         }else{
@@ -4247,10 +4272,12 @@ case OP_NewRowid: {           /* out2 */
 ** sqlite3_last_insert_rowid() function (otherwise it is unmodified).
 **
 ** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of
-** the last seek operation (OP_NotExists) was a success, then this
+** the last seek operation (OP_NotExists or OP_SeekRowid) was a success,
+** then this
 ** operation will not attempt to find the appropriate row before doing
 ** the insert but will instead overwrite the row that the cursor is
-** currently pointing to.  Presumably, the prior OP_NotExists opcode
+** currently pointing to.  Presumably, the prior OP_NotExists or
+** OP_SeekRowid opcode
 ** has already positioned the cursor correctly.  This is an optimization
 ** that boosts performance by avoiding redundant seeks.
 **
@@ -4282,13 +4309,12 @@ case OP_Insert:
 case OP_InsertInt: {
   Mem *pData;       /* MEM cell holding data for the record to be inserted */
   Mem *pKey;        /* MEM cell holding key  for the record */
-  i64 iKey;         /* The integer ROWID or key for the record to be inserted */
   VdbeCursor *pC;   /* Cursor to table into which insert is written */
-  int nZero;        /* Number of zero-bytes to append */
   int seekResult;   /* Result of prior seek or 0 if no USESEEKRESULT flag */
   const char *zDb;  /* database name - used by the update hook */
   Table *pTab;      /* Table structure - used by update and pre-update hooks */
   int op;           /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */
+  BtreePayload x;   /* Payload to be inserted */
 
   op = 0;
   pData = &aMem[pOp->p2];
@@ -4307,10 +4333,10 @@ case OP_InsertInt: {
     assert( pKey->flags & MEM_Int );
     assert( memIsValid(pKey) );
     REGISTER_TRACE(pOp->p3, pKey);
-    iKey = pKey->u.i;
+    x.nKey = pKey->u.i;
   }else{
     assert( pOp->opcode==OP_InsertInt );
-    iKey = pOp->p3;
+    x.nKey = pOp->p3;
   }
 
   if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){
@@ -4331,26 +4357,28 @@ case OP_InsertInt: {
    && pOp->p4type==P4_TABLE
    && !(pOp->p5 & OPFLAG_ISUPDATE)
   ){
-    sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, iKey, pOp->p2);
+    sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey, pOp->p2);
   }
 #endif
 
   if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++;
-  if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = iKey;
+  if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = x.nKey;
   if( pData->flags & MEM_Null ){
-    pData->z = 0;
-    pData->n = 0;
+    x.pData = 0;
+    x.nData = 0;
   }else{
     assert( pData->flags & (MEM_Blob|MEM_Str) );
+    x.pData = pData->z;
+    x.nData = pData->n;
   }
   seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0);
   if( pData->flags & MEM_Zero ){
-    nZero = pData->u.nZero;
+    x.nZero = pData->u.nZero;
   }else{
-    nZero = 0;
+    x.nZero = 0;
   }
-  rc = sqlite3BtreeInsert(pC->uc.pCursor, 0, iKey,
-                          pData->z, pData->n, nZero,
+  x.pKey = 0;
+  rc = sqlite3BtreeInsert(pC->uc.pCursor, &x,
                           (pOp->p5 & OPFLAG_APPEND)!=0, seekResult
   );
   pC->deferredMoveto = 0;
@@ -4359,7 +4387,7 @@ case OP_InsertInt: {
   /* Invoke the update-hook if required. */
   if( rc ) goto abort_due_to_error;
   if( db->xUpdateCallback && op ){
-    db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, iKey);
+    db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, x.nKey);
   }
   break;
 }
@@ -4418,8 +4446,7 @@ case OP_Delete: {
     /* If p5 is zero, the seek operation that positioned the cursor prior to
     ** OP_Delete will have also set the pC->movetoTarget field to the rowid of
     ** the row that is being deleted */
-    i64 iKey = 0;
-    sqlite3BtreeKeySize(pC->uc.pCursor, &iKey);
+    i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor);
     assert( pC->movetoTarget==iKey );
   }
 #endif
@@ -4435,7 +4462,7 @@ case OP_Delete: {
     zDb = db->aDb[pC->iDb].zName;
     pTab = pOp->p4.pTab;
     if( (pOp->p5 & OPFLAG_SAVEPOSITION)!=0 && pC->isTable ){
-      sqlite3BtreeKeySize(pC->uc.pCursor, &pC->movetoTarget);
+      pC->movetoTarget = sqlite3BtreeIntegerKey(pC->uc.pCursor);
     }
   }else{
     zDb = 0;   /* Not needed.  Silence a compiler warning. */
@@ -4589,7 +4616,6 @@ case OP_RowData: {
   VdbeCursor *pC;
   BtCursor *pCrsr;
   u32 n;
-  i64 n64;
 
   pOut = &aMem[pOp->p2];
   memAboutToChange(p, pOut);
@@ -4607,8 +4633,9 @@ case OP_RowData: {
   pCrsr = pC->uc.pCursor;
 
   /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or
-  ** OP_Rewind/Op_Next with no intervening instructions that might invalidate
-  ** the cursor.  If this where not the case, on of the following assert()s
+  ** OP_SeekRowid or OP_Rewind/Op_Next with no intervening instructions
+  ** that might invalidate the cursor.
+  ** If this where not the case, on of the following assert()s
   ** would fail.  Should this ever change (because of changes in the code
   ** generator) then the fix would be to insert a call to
   ** sqlite3VdbeCursorMoveto().
@@ -4620,20 +4647,9 @@ case OP_RowData: {
   if( rc!=SQLITE_OK ) goto abort_due_to_error;
 #endif
 
-  if( pC->isTable==0 ){
-    assert( !pC->isTable );
-    VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &n64);
-    assert( rc==SQLITE_OK );    /* True because of CursorMoveto() call above */
-    if( n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){
-      goto too_big;
-    }
-    n = (u32)n64;
-  }else{
-    VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &n);
-    assert( rc==SQLITE_OK );    /* DataSize() cannot fail */
-    if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
-      goto too_big;
-    }
+  n = sqlite3BtreePayloadSize(pCrsr);
+  if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
+    goto too_big;
   }
   testcase( n==0 );
   if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){
@@ -4698,8 +4714,7 @@ case OP_Rowid: {                 /* out2 */
       pOut->flags = MEM_Null;
       break;
     }
-    rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
-    assert( rc==SQLITE_OK );  /* Always so because of CursorRestore() above */
+    v = sqlite3BtreeIntegerKey(pC->uc.pCursor);
   }
   pOut->u.i = v;
   break;
@@ -4974,8 +4989,7 @@ next_tail:
 case OP_SorterInsert:       /* in2 */
 case OP_IdxInsert: {        /* in2 */
   VdbeCursor *pC;
-  int nKey;
-  const char *zKey;
+  BtreePayload x;
 
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
@@ -4991,9 +5005,12 @@ case OP_IdxInsert: {        /* in2 */
   if( pOp->opcode==OP_SorterInsert ){
     rc = sqlite3VdbeSorterWrite(pC, pIn2);
   }else{
-    nKey = pIn2->n;
-    zKey = pIn2->z;
-    rc = sqlite3BtreeInsert(pC->uc.pCursor, zKey, nKey, "", 0, 0, pOp->p3, 
+    x.nKey = pIn2->n;
+    x.pKey = pIn2->z;
+    x.nData = 0;
+    x.nZero = 0;
+    x.pData = 0;
+    rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, pOp->p3, 
         ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
         );
     assert( pC->deferredMoveto==0 );

src/vdbeapi.c

@@ -1650,8 +1650,7 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
     u32 nRec;
     u8 *aRec;
 
-    rc = sqlite3BtreeDataSize(p->pCsr->uc.pCursor, &nRec);
-    if( rc!=SQLITE_OK ) goto preupdate_old_out;
+    nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor);
     aRec = sqlite3DbMallocRaw(db, nRec);
     if( !aRec ) goto preupdate_old_out;
     rc = sqlite3BtreeData(p->pCsr->uc.pCursor, 0, nRec, aRec);

src/vdbeaux.c

@@ -791,7 +791,7 @@ void sqlite3VdbeJumpHere(Vdbe *p, int addr){
 ** the FuncDef is not ephermal, then do nothing.
 */
 static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){
-  if( ALWAYS(pDef) && (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){
+  if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){
     sqlite3DbFree(db, pDef);
   }
 }
@@ -801,12 +801,20 @@ static void vdbeFreeOpArray(sqlite3 *, Op *, int);
 /*
 ** Delete a P4 value if necessary.
 */
+static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){
+  if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc);
+  sqlite3DbFree(db, p);
+}
+static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){
+  freeEphemeralFunction(db, p->pFunc);
+  sqlite3DbFree(db, p);
+}
 static void freeP4(sqlite3 *db, int p4type, void *p4){
   assert( db );
   switch( p4type ){
     case P4_FUNCCTX: {
-      freeEphemeralFunction(db, ((sqlite3_context*)p4)->pFunc);
-      /* Fall through into the next case */
+      freeP4FuncCtx(db, (sqlite3_context*)p4);
+      break;
     }
     case P4_REAL:
     case P4_INT64:
@@ -837,9 +845,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){
       if( db->pnBytesFreed==0 ){
         sqlite3ValueFree((sqlite3_value*)p4);
       }else{
-        Mem *p = (Mem*)p4;
-        if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc);
-        sqlite3DbFree(db, p);
+        freeP4Mem(db, (Mem*)p4);
       }
       break;
     }
@@ -4309,8 +4315,7 @@ int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){
   ** this code can safely assume that nCellKey is 32-bits  
   */
   assert( sqlite3BtreeCursorIsValid(pCur) );
-  VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey);
-  assert( rc==SQLITE_OK );     /* pCur is always valid so KeySize cannot fail */
+  nCellKey = sqlite3BtreePayloadSize(pCur);
   assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey );
 
   /* Read in the complete content of the index entry */
@@ -4387,8 +4392,7 @@ int sqlite3VdbeIdxKeyCompare(
   assert( pC->eCurType==CURTYPE_BTREE );
   pCur = pC->uc.pCursor;
   assert( sqlite3BtreeCursorIsValid(pCur) );
-  VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey);
-  assert( rc==SQLITE_OK );    /* pCur is always valid so KeySize cannot fail */
+  nCellKey = sqlite3BtreePayloadSize(pCur);
   /* nCellKey will always be between 0 and 0xffffffff because of the way
   ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */
   if( nCellKey<=0 || nCellKey>0x7fffffff ){

src/vdbeblob.c

@@ -415,7 +415,7 @@ static int blobReadWrite(
       ** anyhow.
       */
       sqlite3_int64 iKey;
-      sqlite3BtreeKeySize(p->pCsr, &iKey);
+      iKey = sqlite3BtreeIntegerKey(p->pCsr);
       sqlite3VdbePreUpdateHook(
           v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1
       );

src/vdbemem.c

@@ -991,11 +991,7 @@ int sqlite3VdbeMemFromBtree(
   /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() 
   ** that both the BtShared and database handle mutexes are held. */
   assert( (pMem->flags & MEM_RowSet)==0 );
-  if( key ){
-    zData = (char *)sqlite3BtreeKeyFetch(pCur, &available);
-  }else{
-    zData = (char *)sqlite3BtreeDataFetch(pCur, &available);
-  }
+  zData = (char *)sqlite3BtreePayloadFetch(pCur, &available);
   assert( zData!=0 );
 
   if( offset+amt<=available ){

src/vdbesort.c

@@ -1342,19 +1342,18 @@ static int vdbeSortAllocUnpacked(SortSubtask *pTask){
 
 /*
 ** Merge the two sorted lists p1 and p2 into a single list.
-** Set *ppOut to the head of the new list.
 */
-static void vdbeSorterMerge(
+static SorterRecord *vdbeSorterMerge(
   SortSubtask *pTask,             /* Calling thread context */
   SorterRecord *p1,               /* First list to merge */
-  SorterRecord *p2,               /* Second list to merge */
-  SorterRecord **ppOut            /* OUT: Head of merged list */
+  SorterRecord *p2                /* Second list to merge */
 ){
   SorterRecord *pFinal = 0;
   SorterRecord **pp = &pFinal;
   int bCached = 0;
 
-  while( p1 && p2 ){
+  assert( p1!=0 && p2!=0 );
+  for(;;){
     int res;
     res = pTask->xCompare(
         pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal
@@ -1364,15 +1363,22 @@ static void vdbeSorterMerge(
       *pp = p1;
       pp = &p1->u.pNext;
       p1 = p1->u.pNext;
+      if( p1==0 ){
+        *pp = p2;
+        break;
+      }
     }else{
       *pp = p2;
       pp = &p2->u.pNext;
       p2 = p2->u.pNext;
       bCached = 0;
+      if( p2==0 ){
+        *pp = p1;
+        break;
+      }
     }
   }
-  *pp = p1 ? p1 : p2;
-  *ppOut = pFinal;
+  return pFinal;
 }
 
 /*
@@ -1425,7 +1431,7 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){
 
     p->u.pNext = 0;
     for(i=0; aSlot[i]; i++){
-      vdbeSorterMerge(pTask, p, aSlot[i], &p);
+      p = vdbeSorterMerge(pTask, p, aSlot[i]);
       aSlot[i] = 0;
     }
     aSlot[i] = p;
@@ -1434,7 +1440,8 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){
 
   p = 0;
   for(i=0; i<64; i++){
-    vdbeSorterMerge(pTask, p, aSlot[i], &p);
+    if( aSlot[i]==0 ) continue;
+    p = p ? vdbeSorterMerge(pTask, p, aSlot[i]) : aSlot[i];
   }
   pList->pList = p;
 

src/vtab.c

@@ -418,7 +418,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
     v = sqlite3GetVdbe(pParse);
     sqlite3ChangeCookie(pParse, iDb);
 
-    sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
+    sqlite3VdbeAddOp0(v, OP_Expire);
     zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
     sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
 
@@ -754,10 +754,24 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
      && (pParse->pNewTable->tabFlags & TF_Virtual)==0
     ){
       if( !pTab->aCol ){
-        pTab->aCol = pParse->pNewTable->aCol;
-        pTab->nCol = pParse->pNewTable->nCol;
-        pParse->pNewTable->nCol = 0;
-        pParse->pNewTable->aCol = 0;
+        Table *pNew = pParse->pNewTable;
+        Index *pIdx;
+        pTab->aCol = pNew->aCol;
+        pTab->nCol = pNew->nCol;
+        pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid);
+        pNew->nCol = 0;
+        pNew->aCol = 0;
+        assert( pTab->pIndex==0 );
+        if( !HasRowid(pNew) && pCtx->pVTable->pMod->pModule->xUpdate!=0 ){
+          rc = SQLITE_ERROR;
+        }
+        pIdx = pNew->pIndex;
+        if( pIdx ){
+          assert( pIdx->pNext==0 );
+          pTab->pIndex = pIdx;
+          pNew->pIndex = 0;
+          pIdx->pTable = pTab;
+        }
       }
       pCtx->bDeclared = 1;
     }else{
@@ -1087,7 +1101,7 @@ void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){
 }
 
 /*
-** Check to see if virtual tale module pMod can be have an eponymous
+** Check to see if virtual table module pMod can be have an eponymous
 ** virtual table instance.  If it can, create one if one does not already
 ** exist. Return non-zero if the eponymous virtual table instance exists
 ** when this routine returns, and return zero if it does not exist.
@@ -1104,17 +1118,18 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
   const sqlite3_module *pModule = pMod->pModule;
   Table *pTab;
   char *zErr = 0;
-  int nName;
   int rc;
   sqlite3 *db = pParse->db;
   if( pMod->pEpoTab ) return 1;
   if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0;
-  nName = sqlite3Strlen30(pMod->zName) + 1;
-  pTab = sqlite3DbMallocZero(db, sizeof(Table) + nName);
+  pTab = sqlite3DbMallocZero(db, sizeof(Table));
   if( pTab==0 ) return 0;
+  pTab->zName = sqlite3DbStrDup(db, pMod->zName);
+  if( pTab->zName==0 ){
+    sqlite3DbFree(db, pTab);
+    return 0;
+  }
   pMod->pEpoTab = pTab;
-  pTab->zName = (char*)&pTab[1];
-  memcpy(pTab->zName, pMod->zName, nName);
   pTab->nRef = 1;
   pTab->pSchema = db->aDb[0].pSchema;
   pTab->tabFlags |= TF_Virtual;
@@ -1140,9 +1155,11 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
 void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){
   Table *pTab = pMod->pEpoTab;
   if( pTab!=0 ){
-    sqlite3DeleteColumnNames(db, pTab);
-    sqlite3VtabClear(db, pTab);
-    sqlite3DbFree(db, pTab);
+    /* Mark the table as Ephemeral prior to deleting it, so that the
+    ** sqlite3DeleteTable() routine will know that it is not stored in 
+    ** the schema. */
+    pTab->tabFlags |= TF_Ephemeral;
+    sqlite3DeleteTable(db, pTab);
     pMod->pEpoTab = 0;
   }
 }

src/wal.c

@@ -3109,16 +3109,21 @@ int sqlite3WalFrames(
   ** past the sector boundary is written after the sync.
   */
   if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){
+    int bSync = 1;
     if( pWal->padToSectorBoundary ){
       int sectorSize = sqlite3SectorSize(pWal->pWalFd);
       w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize;
+      bSync = (w.iSyncPoint==iOffset);
+      testcase( bSync );
       while( iOffset<w.iSyncPoint ){
         rc = walWriteOneFrame(&w, pLast, nTruncate, iOffset);
         if( rc ) return rc;
         iOffset += szFrame;
         nExtra++;
       }
-    }else{
+    }
+    if( bSync ){
+      assert( rc==SQLITE_OK );
       rc = sqlite3OsSync(w.pFd, sync_flags & SQLITE_SYNC_MASK);
     }
   }

src/where.c

@@ -51,6 +51,18 @@ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
   return pWInfo->nOBSat;
 }
 
+/*
+** Return TRUE if the innermost loop of the WHERE clause implementation
+** returns rows in ORDER BY order for complete run of the inner loop.
+**
+** Across multiple iterations of outer loops, the output rows need not be
+** sorted.  As long as rows are sorted for just the innermost loop, this
+** routine can return TRUE.
+*/
+int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){
+  return pWInfo->bOrderedInnerLoop;
+}
+
 /*
 ** Return the VDBE address or label to jump to in order to continue
 ** immediately with the next row of a WHERE clause.
@@ -1636,19 +1648,41 @@ static void whereTermPrint(WhereTerm *pTerm, int iTerm){
     sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
   }else{
     char zType[4];
+    char zLeft[50];
     memcpy(zType, "...", 4);
     if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
     if( pTerm->eOperator & WO_EQUIV  ) zType[1] = 'E';
     if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L';
+    if( pTerm->eOperator & WO_SINGLE ){
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
+                       pTerm->leftCursor, pTerm->u.leftColumn);
+    }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%lld", 
+                       pTerm->u.pOrInfo->indexable);
+    }else{
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
+    }
     sqlite3DebugPrintf(
-       "TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x wtFlags=0x%04x\n",
-       iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb,
+       "TERM-%-3d %p %s %-12s prob=%-3d op=0x%03x wtFlags=0x%04x\n",
+       iTerm, pTerm, zType, zLeft, pTerm->truthProb,
        pTerm->eOperator, pTerm->wtFlags);
     sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
   }
 }
 #endif
 
+#ifdef WHERETRACE_ENABLED
+/*
+** Show the complete content of a WhereClause
+*/
+void sqlite3WhereClausePrint(WhereClause *pWC){
+  int i;
+  for(i=0; i<pWC->nTerm; i++){
+    whereTermPrint(&pWC->a[i], i);
+  }
+}
+#endif
+
 #ifdef WHERETRACE_ENABLED
 /*
 ** Print a WhereLoop object for debugging purposes
@@ -2635,7 +2669,7 @@ static int whereLoopAddBtree(
 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
   /* Automatic indexes */
   if( !pBuilder->pOrSet      /* Not part of an OR optimization */
-   && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0
+   && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0
    && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
    && pSrc->pIBIndex==0      /* Has no INDEXED BY clause */
    && !pSrc->fg.notIndexed   /* Has no NOT INDEXED clause */
@@ -2667,6 +2701,7 @@ static int whereLoopAddBtree(
           pNew->rSetup += 24;
         }
         ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
+        if( pNew->rSetup<0 ) pNew->rSetup = 0;
         /* TUNING: Each index lookup yields 20 rows in the table.  This
         ** is more than the usual guess of 10 rows, since we have no way
         ** of knowing how selective the index will ultimately be.  It would
@@ -2727,6 +2762,7 @@ static int whereLoopAddBtree(
       /* Full scan via index */
       if( b
        || !HasRowid(pTab)
+       || pProbe->pPartIdxWhere!=0
        || ( m==0
          && pProbe->bUnordered==0
          && (pProbe->szIdxRow<pTab->szTabRow)
@@ -3112,9 +3148,7 @@ static int whereLoopAddOr(
         WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", 
                    (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
         if( sqlite3WhereTrace & 0x400 ){
-          for(i=0; i<sSubBuild.pWC->nTerm; i++){
-            whereTermPrint(&sSubBuild.pWC->a[i], i);
-          }
+          sqlite3WhereClausePrint(sSubBuild.pWC);
         }
 #endif
 #ifndef SQLITE_OMIT_VIRTUALTABLE
@@ -3207,6 +3241,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
       mPrereq = mPrior;
     }
     priorJointype = pItem->fg.jointype;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
     if( IsVirtual(pItem->pTab) ){
       struct SrcList_item *p;
       for(p=&pItem[1]; p<pEnd; p++){
@@ -3215,7 +3250,9 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
         }
       }
       rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
-    }else{
+    }else
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+    {
       rc = whereLoopAddBtree(pBuilder, mPrereq);
     }
     if( rc==SQLITE_OK ){
@@ -3250,7 +3287,7 @@ static i8 wherePathSatisfiesOrderBy(
   WhereInfo *pWInfo,    /* The WHERE clause */
   ExprList *pOrderBy,   /* ORDER BY or GROUP BY or DISTINCT clause to check */
   WherePath *pPath,     /* The WherePath to check */
-  u16 wctrlFlags,       /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */
+  u16 wctrlFlags,       /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
   u16 nLoop,            /* Number of entries in pPath->aLoop[] */
   WhereLoop *pLast,     /* Add this WhereLoop to the end of pPath->aLoop[] */
   Bitmask *pRevMask     /* OUT: Mask of WhereLoops to run in reverse order */
@@ -3261,6 +3298,7 @@ static i8 wherePathSatisfiesOrderBy(
   u8 isOrderDistinct;   /* All prior WhereLoops are order-distinct */
   u8 distinctColumns;   /* True if the loop has UNIQUE NOT NULL columns */
   u8 isMatch;           /* iColumn matches a term of the ORDER BY clause */
+  u16 eqOpMask;         /* Allowed equality operators */
   u16 nKeyCol;          /* Number of key columns in pIndex */
   u16 nColumn;          /* Total number of ordered columns in the index */
   u16 nOrderBy;         /* Number terms in the ORDER BY clause */
@@ -3311,9 +3349,16 @@ static i8 wherePathSatisfiesOrderBy(
   obDone = MASKBIT(nOrderBy)-1;
   orderDistinctMask = 0;
   ready = 0;
+  eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
+  if( wctrlFlags & WHERE_ORDERBY_LIMIT ) eqOpMask |= WO_IN;
   for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){
     if( iLoop>0 ) ready |= pLoop->maskSelf;
-    pLoop = iLoop<nLoop ? pPath->aLoop[iLoop] : pLast;
+    if( iLoop<nLoop ){
+      pLoop = pPath->aLoop[iLoop];
+      if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
+    }else{
+      pLoop = pLast;
+    }
     if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
       if( pLoop->u.vtab.isOrdered ) obSat = obDone;
       break;
@@ -3331,7 +3376,7 @@ static i8 wherePathSatisfiesOrderBy(
       if( pOBExpr->op!=TK_COLUMN ) continue;
       if( pOBExpr->iTable!=iCur ) continue;
       pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
-                       ~ready, WO_EQ|WO_ISNULL|WO_IS, 0);
+                       ~ready, eqOpMask, 0);
       if( pTerm==0 ) continue;
       if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
         const char *z1, *z2;
@@ -3371,10 +3416,12 @@ static i8 wherePathSatisfiesOrderBy(
       for(j=0; j<nColumn; j++){
         u8 bOnce;   /* True to run the ORDER BY search loop */
 
-        /* Skip over == and IS NULL terms */
+        /* Skip over == and IS and ISNULL terms.
+        ** (Also skip IN terms when doing WHERE_ORDERBY_LIMIT processing)
+        */
         if( j<pLoop->u.btree.nEq
          && pLoop->nSkip==0
-         && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL|WO_IS))!=0
+         && ((i = pLoop->aLTerm[j]->eOperator) & eqOpMask)!=0
         ){
           if( i & WO_ISNULL ){
             testcase( isOrderDistinct );
@@ -3898,8 +3945,19 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
       }
     }else{
       pWInfo->nOBSat = pFrom->isOrdered;
-      if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0;
       pWInfo->revMask = pFrom->revLoop;
+      if( pWInfo->nOBSat<=0 ){
+        pWInfo->nOBSat = 0;
+        if( nLoop>0 ){
+          Bitmask m;
+          int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
+                      WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
+          if( rc==pWInfo->pOrderBy->nExpr ){
+            pWInfo->bOrderedInnerLoop = 1;
+            pWInfo->revMask = m;
+          }
+        }
+      }
     }
     if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
         && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
@@ -3947,7 +4005,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   Index *pIdx;
 
   pWInfo = pBuilder->pWInfo;
-  if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0;
+  if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
   assert( pWInfo->pTabList->nSrc>=1 );
   pItem = pWInfo->pTabList->a;
   pTab = pItem->pTab;
@@ -4094,7 +4152,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
 **
 ** The iIdxCur parameter is the cursor number of an index.  If 
-** WHERE_ONETABLE_ONLY is set, iIdxCur is the cursor number of an index
+** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
 ** to use for OR clause processing.  The WHERE clause should use this
 ** specific cursor.  If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
 ** the first cursor in an array of cursors for all indices.  iIdxCur should
@@ -4108,7 +4166,7 @@ WhereInfo *sqlite3WhereBegin(
   ExprList *pOrderBy,     /* An ORDER BY (or GROUP BY) clause, or NULL */
   ExprList *pDistinctSet, /* Try not to output two rows that duplicate these */
   u16 wctrlFlags,         /* The WHERE_* flags defined in sqliteInt.h */
-  int iAuxArg             /* If WHERE_ONETABLE_ONLY is set, index cursor number
+  int iAuxArg             /* If WHERE_OR_SUBCLAUSE is set, index cursor number
                           ** If WHERE_USE_LIMIT, then the limit amount */
 ){
   int nByteWInfo;            /* Num. bytes allocated for WhereInfo struct */
@@ -4127,11 +4185,11 @@ WhereInfo *sqlite3WhereBegin(
 
   assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
         (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 
-     && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 
+     && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 
   ));
 
-  /* Only one of WHERE_ONETABLE_ONLY or WHERE_USE_LIMIT */
-  assert( (wctrlFlags & WHERE_ONETABLE_ONLY)==0
+  /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
+  assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
             || (wctrlFlags & WHERE_USE_LIMIT)==0 );
 
   /* Variable initialization */
@@ -4159,11 +4217,11 @@ WhereInfo *sqlite3WhereBegin(
   }
 
   /* This function normally generates a nested loop for all tables in 
-  ** pTabList.  But if the WHERE_ONETABLE_ONLY flag is set, then we should
+  ** pTabList.  But if the WHERE_OR_SUBCLAUSE flag is set, then we should
   ** only generate code for the first table in pTabList and assume that
   ** any cursors associated with subsequent tables are uninitialized.
   */
-  nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc;
+  nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
 
   /* Allocate and initialize the WhereInfo structure that will become the
   ** return value. A single allocation is used to store the WhereInfo
@@ -4239,7 +4297,7 @@ WhereInfo *sqlite3WhereBegin(
   ** Note that bitmasks are created for all pTabList->nSrc tables in
   ** pTabList, not just the first nTabList tables.  nTabList is normally
   ** equal to pTabList->nSrc but might be shortened to 1 if the
-  ** WHERE_ONETABLE_ONLY flag is set.
+  ** WHERE_OR_SUBCLAUSE flag is set.
   */
   for(ii=0; ii<pTabList->nSrc; ii++){
     createMask(pMaskSet, pTabList->a[ii].iCursor);
@@ -4277,10 +4335,7 @@ WhereInfo *sqlite3WhereBegin(
     sqlite3DebugPrintf(")\n");
   }
   if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */
-    int i;
-    for(i=0; i<sWLB.pWC->nTerm; i++){
-      whereTermPrint(&sWLB.pWC->a[i], i);
-    }
+    sqlite3WhereClausePrint(sWLB.pWC);
   }
 #endif
 
@@ -4422,7 +4477,7 @@ WhereInfo *sqlite3WhereBegin(
     }else
 #endif
     if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
-         && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){
+         && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){
       int op = OP_OpenRead;
       if( pWInfo->eOnePass!=ONEPASS_OFF ){
         op = OP_OpenWrite;
@@ -4461,7 +4516,7 @@ WhereInfo *sqlite3WhereBegin(
       /* iAuxArg is always set if to a positive value if ONEPASS is possible */
       assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
       if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
-       && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0
+       && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
       ){
         /* This is one term of an OR-optimization using the PRIMARY KEY of a
         ** WITHOUT ROWID table.  No need for a separate index */
@@ -4477,7 +4532,7 @@ WhereInfo *sqlite3WhereBegin(
         }
         op = OP_OpenWrite;
         pWInfo->aiCurOnePass[1] = iIndexCur;
-      }else if( iAuxArg && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){
+      }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
         iIndexCur = iAuxArg;
         op = OP_ReopenIdx;
       }else{
@@ -4541,7 +4596,7 @@ WhereInfo *sqlite3WhereBegin(
     pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
     notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady);
     pWInfo->iContinue = pLevel->addrCont;
-    if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){
+    if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
       sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
     }
   }
@@ -4664,12 +4719,12 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
 
     /* Close all of the cursors that were opened by sqlite3WhereBegin.
     ** Except, do not close cursors that will be reused by the OR optimization
-    ** (WHERE_OMIT_OPEN_CLOSE).  And do not close the OP_OpenWrite cursors
+    ** (WHERE_OR_SUBCLAUSE).  And do not close the OP_OpenWrite cursors
     ** created for the ONEPASS optimization.
     */
     if( (pTab->tabFlags & TF_Ephemeral)==0
      && pTab->pSelect==0
-     && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0
+     && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0
     ){
       int ws = pLoop->wsFlags;
       if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){

src/whereInt.h

@@ -418,8 +418,9 @@ struct WhereInfo {
   u8 sorted;                /* True if really sorted (not just grouped) */
   u8 eOnePass;              /* ONEPASS_OFF, or _SINGLE, or _MULTI */
   u8 untestedTerms;         /* Not all WHERE terms resolved by outer loop */
-  u8 eDistinct;             /* One of the WHERE_DISTINCT_* values below */
+  u8 eDistinct;             /* One of the WHERE_DISTINCT_* values */
   u8 nLevel;                /* Number of nested loop */
+  u8 bOrderedInnerLoop;     /* True if only the inner-most loop is ordered */
   int iTop;                 /* The very beginning of the WHERE loop */
   int iContinue;            /* Jump here to continue with next record */
   int iBreak;               /* Jump here to break out of the loop */
@@ -436,6 +437,9 @@ struct WhereInfo {
 ** where.c:
 */
 Bitmask sqlite3WhereGetMask(WhereMaskSet*,int);
+#ifdef WHERETRACE_ENABLED
+void sqlite3WhereClausePrint(WhereClause *pWC);
+#endif
 WhereTerm *sqlite3WhereFindTerm(
   WhereClause *pWC,     /* The WHERE clause to be searched */
   int iCur,             /* Cursor number of LHS */

src/wherecode.c

@@ -126,7 +126,7 @@ int sqlite3WhereExplainOneScan(
 
     pLoop = pLevel->pWLoop;
     flags = pLoop->wsFlags;
-    if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0;
+    if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_OR_SUBCLAUSE) ) return 0;
 
     isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
             || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0))
@@ -626,6 +626,38 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){
   return WRC_Continue;
 }
 
+/*
+** Test whether or not expression pExpr, which was part of a WHERE clause,
+** should be included in the cursor-hint for a table that is on the rhs
+** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the 
+** expression is not suitable.
+**
+** An expression is unsuitable if it might evaluate to non NULL even if
+** a TK_COLUMN node that does affect the value of the expression is set
+** to NULL. For example:
+**
+**   col IS NULL
+**   col IS NOT NULL
+**   coalesce(col, 1)
+**   CASE WHEN col THEN 0 ELSE 1 END
+*/
+static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){
+  if( pExpr->op==TK_IS 
+   || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT 
+   || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE 
+  ){
+    pWalker->eCode = 1;
+  }else if( pExpr->op==TK_FUNCTION ){
+    int d1;
+    char d2[3];
+    if( 0==sqlite3IsLikeFunction(pWalker->pParse->db, pExpr, &d1, d2) ){
+      pWalker->eCode = 1;
+    }
+  }
+
+  return WRC_Continue;
+}
+
 
 /*
 ** This function is called on every node of an expression tree used as an
@@ -678,6 +710,7 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
 ** Insert an OP_CursorHint instruction if it is appropriate to do so.
 */
 static void codeCursorHint(
+  struct SrcList_item *pTabItem,  /* FROM clause item */
   WhereInfo *pWInfo,    /* The where clause */
   WhereLevel *pLevel,   /* Which loop to provide hints for */
   WhereTerm *pEndRange  /* Hint this end-of-scan boundary term if not NULL */
@@ -708,7 +741,42 @@ static void codeCursorHint(
     pTerm = &pWC->a[i];
     if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
     if( pTerm->prereqAll & pLevel->notReady ) continue;
-    if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
+
+    /* Any terms specified as part of the ON(...) clause for any LEFT 
+    ** JOIN for which the current table is not the rhs are omitted
+    ** from the cursor-hint. 
+    **
+    ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms 
+    ** that were specified as part of the WHERE clause must be excluded.
+    ** This is to address the following:
+    **
+    **   SELECT ... t1 LEFT JOIN t2 ON (t1.a=t2.b) WHERE t2.c IS NULL;
+    **
+    ** Say there is a single row in t2 that matches (t1.a=t2.b), but its
+    ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is 
+    ** pushed down to the cursor, this row is filtered out, causing
+    ** SQLite to synthesize a row of NULL values. Which does match the
+    ** WHERE clause, and so the query returns a row. Which is incorrect.
+    **
+    ** For the same reason, WHERE terms such as:
+    **
+    **   WHERE 1 = (t2.c IS NULL)
+    **
+    ** are also excluded. See codeCursorHintIsOrFunction() for details.
+    */
+    if( pTabItem->fg.jointype & JT_LEFT ){
+      Expr *pExpr = pTerm->pExpr;
+      if( !ExprHasProperty(pExpr, EP_FromJoin) 
+       || pExpr->iRightJoinTable!=pTabItem->iCursor
+      ){
+        sWalker.eCode = 0;
+        sWalker.xExprCallback = codeCursorHintIsOrFunction;
+        sqlite3WalkExpr(&sWalker, pTerm->pExpr);
+        if( sWalker.eCode ) continue;
+      }
+    }else{
+      if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
+    }
 
     /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize
     ** the cursor.  These terms are not needed as hints for a pure range
@@ -742,7 +810,7 @@ static void codeCursorHint(
   }
 }
 #else
-# define codeCursorHint(A,B,C)  /* No-op */
+# define codeCursorHint(A,B,C,D)  /* No-op */
 #endif /* SQLITE_ENABLE_CURSOR_HINTS */
 
 /*
@@ -776,7 +844,7 @@ static void codeDeferredSeek(
   assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 );
   
   sqlite3VdbeAddOp3(v, OP_Seek, iIdxCur, 0, iCur);
-  if( (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)
+  if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)
    && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask)
   ){
     int i;
@@ -831,7 +899,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur);
   bRev = (pWInfo->revMask>>iLevel)&1;
   omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 
-           && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0;
+           && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0;
   VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName));
 
   /* Create labels for the "break" and "continue" instructions
@@ -971,8 +1039,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg);
     if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg);
     addrNxt = pLevel->addrNxt;
-    sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v);
-    sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg);
+    sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg);
     VdbeCoverage(v);
     sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1);
     sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
@@ -999,7 +1066,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       pStart = pEnd;
       pEnd = pTerm;
     }
-    codeCursorHint(pWInfo, pLevel, pEnd);
+    codeCursorHint(pTabItem, pWInfo, pLevel, pEnd);
     if( pStart ){
       Expr *pX;             /* The expression that defines the start bound */
       int r1, rTemp;        /* Registers for holding the start boundary */
@@ -1213,7 +1280,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     ** and store the values of those terms in an array of registers
     ** starting at regBase.
     */
-    codeCursorHint(pWInfo, pLevel, pRangeEnd);
+    codeCursorHint(pTabItem, pWInfo, pLevel, pRangeEnd);
     regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff);
     assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq );
     if( zStartAff ) cEndAff = zStartAff[nEq];
@@ -1517,11 +1584,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     ** eliminating duplicates from other WHERE clauses, the action for each
     ** sub-WHERE clause is to to invoke the main loop body as a subroutine.
     */
-    wctrlFlags =  WHERE_OMIT_OPEN_CLOSE
-                | WHERE_FORCE_TABLE
-                | WHERE_ONETABLE_ONLY
-                | WHERE_NO_AUTOINDEX
-                | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE);
+    wctrlFlags =  WHERE_OR_SUBCLAUSE | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE);
     for(ii=0; ii<pOrWc->nTerm; ii++){
       WhereTerm *pOrTerm = &pOrWc->a[ii];
       if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){
@@ -1665,7 +1728,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       ** a pseudo-cursor.  No need to Rewind or Next such cursors. */
       pLevel->op = OP_Noop;
     }else{
-      codeCursorHint(pWInfo, pLevel, 0);
+      codeCursorHint(pTabItem, pWInfo, pLevel, 0);
       pLevel->op = aStep[bRev];
       pLevel->p1 = iCur;
       pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);
@@ -1690,7 +1753,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
     if( (pTerm->prereqAll & pLevel->notReady)!=0 ){
       testcase( pWInfo->untestedTerms==0
-               && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 );
+               && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 );
       pWInfo->untestedTerms = 1;
       continue;
     }

src/whereexpr.c

@@ -568,7 +568,9 @@ static void exprAnalyzeOrTerm(
         if( !db->mallocFailed ){
           for(j=0, pAndTerm=pAndWC->a; j<pAndWC->nTerm; j++, pAndTerm++){
             assert( pAndTerm->pExpr );
-            if( allowedOp(pAndTerm->pExpr->op) ){
+            if( allowedOp(pAndTerm->pExpr->op) 
+             || pAndTerm->eOperator==WO_MATCH 
+            ){
               b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor);
             }
           }
@@ -1122,7 +1124,7 @@ static void exprAnalyze(
   ** virtual tables.  The native query optimizer does not attempt
   ** to do anything with MATCH functions.
   */
-  if( isMatchOfColumn(pExpr, &eOp2) ){
+  if( pWC->op==TK_AND && isMatchOfColumn(pExpr, &eOp2) ){
     int idxNew;
     Expr *pRight, *pLeft;
     WhereTerm *pNewTerm;

test/bestindex3.test

@@ -0,0 +1,170 @@
+# 2016 May 29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindex3
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Virtual table callback for a virtual table named $tbl.
+#
+# The table created is:
+#
+#      "CREATE TABLE t1 (a, b, c)"
+#
+# This virtual table supports both LIKE and = operators on all columns.
+#  
+proc vtab_cmd {bOmit method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c)"
+    }
+
+    xBestIndex {
+      foreach {clist orderby mask} $args {}
+
+      set ret [list]
+      set use use
+      if {$bOmit} {set use omit}
+
+      for {set i 0} {$i < [llength $clist]} {incr i} {
+        array unset C
+        array set C [lindex $clist $i]
+        if {$C(usable) && ($C(op)=="like" || $C(op)=="eq")} {
+          lappend ret $use $i
+          lappend ret idxstr 
+          lappend ret "[lindex {a b c} $C(column)] [string toupper $C(op)] ?"
+          break
+        }
+      }
+
+      if {$ret==""} {
+        lappend ret cost 1000000 rows 1000000
+      } else {
+        lappend ret cost 100 rows 10
+      }
+      return $ret
+    }
+
+    xFilter {
+      foreach {idxnum idxstr param} $args {}
+      set where ""
+      if {$bOmit && $idxstr != ""} {
+        set where " WHERE [string map [list ? '$param' EQ =] $idxstr]"
+      }
+      return [list sql "SELECT rowid, * FROM ttt$where"]
+    }
+  }
+  return ""
+}
+
+register_tcl_module db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING tcl("vtab_cmd 0");
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM t1 WHERE a LIKE 'abc';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
+}
+
+do_eqp_test 1.2 {
+  SELECT * FROM t1 WHERE a = 'abc';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
+}
+
+do_eqp_test 1.3 {
+  SELECT * FROM t1 WHERE a = 'abc' OR b = 'def';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
+}
+
+do_eqp_test 1.4 {
+  SELECT * FROM t1 WHERE a LIKE 'abc%' OR b = 'def';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
+}
+
+do_execsql_test 1.5 {
+  CREATE TABLE ttt(a, b, c);
+
+  INSERT INTO ttt VALUES(1, 'two',   'three');
+  INSERT INTO ttt VALUES(2, 'one',   'two');
+  INSERT INTO ttt VALUES(3, 'three', 'one');
+  INSERT INTO ttt VALUES(4, 'y',     'one');
+  INSERT INTO ttt VALUES(5, 'x',     'two');
+  INSERT INTO ttt VALUES(6, 'y',     'three');
+}
+
+foreach omit {0 1} {
+  do_execsql_test 1.6.$omit.0 "
+    DROP TABLE t1;
+    CREATE VIRTUAL TABLE t1 USING tcl('vtab_cmd $omit');
+  "
+  do_execsql_test 1.6.$omit.1 { 
+    SELECT rowid FROM t1 WHERE c LIKE 'o%'
+  } {3 4}
+
+  do_execsql_test 1.6.$omit.2 { 
+    SELECT rowid FROM t1 WHERE c LIKE 'o%' OR b='y'
+  } {3 4 6}
+
+  do_execsql_test 1.6.$omit.3 { 
+    SELECT rowid FROM t1 WHERE c = 'three' OR c LIKE 'o%'
+  } {1 6 3 4}
+}
+
+#-------------------------------------------------------------------------
+# Test the same pattern works with ordinary tables.
+#
+do_execsql_test 2.1 {
+  CREATE TABLE t2(x TEXT COLLATE nocase, y TEXT);
+  CREATE INDEX t2x ON t2(x COLLATE nocase);
+  CREATE INDEX t2y ON t2(y);
+}
+
+do_eqp_test 2.2 {
+  SELECT * FROM t2 WHERE x LIKE 'abc%' OR y = 'def'
+} {
+  0 0 0 {SEARCH TABLE t2 USING INDEX t2x (x>? AND x<?)}
+  0 0 0 {SEARCH TABLE t2 USING INDEX t2y (y=?)}
+}
+
+#-------------------------------------------------------------------------
+# Test that any PRIMARY KEY within a sqlite3_decl_vtab() CREATE TABLE 
+# statement is currently ignored.
+#
+proc vvv_command {method args} {
+  switch -- $method {
+    xConnect { return "CREATE TABLE t1(a PRIMARY KEY, b, c)" }
+  }
+}
+proc yyy_command {method args} {
+  switch -- $method {
+    xConnect { return "CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b))" }
+  }
+}
+
+do_execsql_test 3.1 { CREATE VIRTUAL TABLE t3 USING tcl('vvv_command') }
+do_execsql_test 3.2 { CREATE VIRTUAL TABLE t4 USING tcl('yyy_command') }
+
+finish_test
+

test/csv01.test

@@ -0,0 +1,112 @@
+# 2016-06-02
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# Test cases for CSV virtual table.
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix closure01
+
+ifcapable !vtab||!cte { finish_test ; return }
+
+load_static_extension db csv
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE temp.t1 USING csv(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4
+  );
+  SELECT * FROM t1 WHERE c1=10;
+} {9 10 11 12}
+do_execsql_test 1.1 {
+  SELECT * FROM t1 WHERE c1='10';
+} {9 10 11 12}
+do_execsql_test 1.2 {
+  SELECT rowid FROM t1;
+} {1 2 3 4}
+
+do_execsql_test 2.0 {
+  DROP TABLE t1;
+  CREATE VIRTUAL TABLE temp.t2 USING csv(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4,
+    schema='CREATE TABLE t2(a INT, b TEXT, c REAL, d BLOB)'
+  );
+  SELECT * FROM t2 WHERE a=9;
+} {9 10 11 12}
+do_execsql_test 2.1 {
+  SELECT * FROM t2 WHERE b=10;
+} {9 10 11 12}
+do_execsql_test 2.2 {
+  SELECT * FROM t2 WHERE c=11;
+} {9 10 11 12}
+do_execsql_test 2.3 {
+  SELECT * FROM t2 WHERE d=12;
+} {}
+do_execsql_test 2.4 {
+  SELECT * FROM t2 WHERE d='12';
+} {9 10 11 12}
+do_execsql_test 2.5 {
+  SELECT * FROM t2 WHERE a='9';
+} {9 10 11 12}
+
+do_execsql_test 3.0 {
+  DROP TABLE t2;
+  CREATE VIRTUAL TABLE temp.t3 USING csv(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4,
+    schema=
+      'CREATE TABLE t3(a PRIMARY KEY,b TEXT,c TEXT,d TEXT) WITHOUT ROWID',
+    testflags=1
+  );
+  SELECT a FROM t3 WHERE b=6 OR c=7 OR d=12 ORDER BY +a;
+} {5 9}
+do_execsql_test 3.1 {
+  SELECT a FROM t3 WHERE +b=6 OR c=7 OR d=12 ORDER BY +a;
+} {5 9}
+
+# The rowid column is not visible on a WITHOUT ROWID virtual table
+do_catchsql_test 3.2 {
+  SELECT rowid, a FROM t3;
+} {1 {no such column: rowid}}
+
+do_catchsql_test 4.0 {
+  DROP TABLE t3;
+  CREATE VIRTUAL TABLE temp.t4 USING csv_wr(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4,
+    schema=
+      'CREATE TABLE t3(a PRIMARY KEY,b TEXT,c TEXT,d TEXT) WITHOUT ROWID',
+    testflags=1
+  );
+} {1 {vtable constructor failed: t4}}
+
+finish_test

test/ctime.test

@@ -196,9 +196,10 @@ set tc 1
 foreach opt $opts {
   do_test ctime-2.5.$tc {
     set N [ expr {$tc-1} ]
-    set ans1 [ catchsql {
+    set ans1 [catch {db one {
       SELECT sqlite_compileoption_get($N);
-    } ]
+    }} msg]
+    lappend ans1 $msg
     set ans2 [ catchsql {
       SELECT sqlite_compileoption_used($opt);
     } ]

test/cursorhint2.test

@@ -0,0 +1,180 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library. The
+# focus is on testing that cursor-hints are correct for queries
+# involving LEFT JOIN.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix cursorhint2
+
+ifcapable !cursorhints {
+  finish_test
+  return
+}
+
+proc extract_hints {sql} {
+
+  db eval "SELECT tbl_name, rootpage FROM sqlite_master where rootpage" {
+    set lookup($rootpage) $tbl_name
+  }
+
+  set ret [list]
+  db eval "EXPLAIN $sql" a {
+    switch -- $a(opcode) {
+      OpenRead {
+        set csr($a(p1)) $lookup($a(p2))
+      }
+      CursorHint { 
+        lappend ret $csr($a(p1)) $a(p4) 
+      }
+    }
+  }
+
+  set ret
+}
+
+proc do_extract_hints_test {tn sql ret} {
+  uplevel [list do_test $tn [list extract_hints $sql] [list {*}$ret]]
+}
+
+do_execsql_test 1.0 {
+  PRAGMA automatic_index = 0;
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+  CREATE TABLE t3(e, f);
+}
+
+do_extract_hints_test 1.1 {
+  SELECT * FROM t1 WHERE a=1;
+} {
+  t1 EQ(c0,1)
+}
+
+do_extract_hints_test 1.2 {
+  SELECT * FROM t1 CROSS JOIN t2 ON (a=c) WHERE d IS NULL;
+} {
+  t2 {AND(ISNULL(c1),EQ(r[1],c0))}
+}
+
+do_extract_hints_test 1.3 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c) WHERE d IS NULL;
+} {
+  t2 {EQ(r[2],c0)}
+}
+
+do_extract_hints_test 1.4 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND a=10) WHERE d IS NULL;
+} {
+  t2 {AND(EQ(r[2],c0),EQ(r[3],10))}
+}
+
+do_extract_hints_test 1.5 {
+  SELECT * FROM t1 CROSS JOIN t2 ON (a=c AND a=10) WHERE d IS NULL;
+} {
+  t1 EQ(c0,10) t2 {AND(ISNULL(c1),EQ(r[3],c0))}
+}
+
+do_extract_hints_test 1.6 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c) LEFT JOIN t3 ON (d=f);
+} {
+  t2 {EQ(r[2],c0)} t3 {EQ(r[6],c1)}
+}
+
+do_extract_hints_test 1.7 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND d=e) LEFT JOIN t3 ON (d=f);
+} {
+  t2 {EQ(r[2],c0)} t3 {AND(EQ(r[6],c0),EQ(r[7],c1))}
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+  CREATE TABLE x1(x, y);
+  CREATE TABLE x2(a, b);
+}
+
+do_extract_hints_test 2.1 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NULL;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.2 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS +NULL;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.3 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = (b IS NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.4 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = coalesce(b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.5 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = coalesce(b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.6 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.7 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT +NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.8 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NOT +NULL
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.9 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE CASE b WHEN 0 THEN 0 ELSE 1 END;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.10 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b = 32+32
+} {
+  x2 {AND(EQ(c1,ADD(32,32)),EQ(c0,r[2]))}
+}
+
+do_extract_hints_test 2.11 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b LIKE 'abc%'
+} {
+  x2 {AND(expr,EQ(c0,r[2]))}
+}
+
+do_extract_hints_test 2.11 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE coalesce(x2.b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+finish_test
+

test/fts3expr.test

@@ -514,4 +514,8 @@ do_test fts3expr-9.1 {
   test_fts3expr "f (e NEAR/2 a)"
 } {AND {PHRASE 3 0 f} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}}
 
+do_test fts3expr-10.1 { test_fts3expr "abc *" } {PHRASE 3 0 abc}
+do_test fts3expr-10.2 { test_fts3expr "*" } {}
+do_test fts3expr-10.3 { test_fts3expr "abc*" } {PHRASE 3 0 abc+}
+
 finish_test

test/index6.test

@@ -376,4 +376,15 @@ do_execsql_test index6-10.3eqp {
   SELECT e FROM t10 WHERE a=1 AND b=2 ORDER BY d DESC;
 } {~/USING INDEX t10x/}
 
+# A partial index will be used for a full table scan, where possible
+do_execsql_test index6-11.1 {
+  CREATE TABLE t11(a,b,c);
+  CREATE INDEX t11x ON t11(a) WHERE b<>99;
+  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
+} {/USING INDEX t11x/}
+do_execsql_test index6-11.2 {
+  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
+} {/USING INDEX t11x/}
+  
+
 finish_test

test/limit2.test

@@ -0,0 +1,82 @@
+# 2016-05-20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the LIMIT in combination with ORDER BY
+# and in particular, the optimizations in the inner loop that cause an
+# early exit of the inner loop when the LIMIT is reached and the inner
+# loop is emitting rows in ORDER BY order.
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test limit2-100 {
+  CREATE TABLE t1(a,b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
+    INSERT INTO t1(a,b) SELECT 1, (x*17)%1000 + 1000 FROM c;
+  INSERT INTO t1(a,b) VALUES(2,2),(3,1006),(4,4),(5,9999);
+  CREATE INDEX t1ab ON t1(a,b);
+}
+set sqlite_search_count 0
+do_execsql_test limit2-100.1 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+set fast_count $sqlite_search_count
+set sqlite_search_count 0
+do_execsql_test limit2-100.2 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY +b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+do_test limit2-100.3 {
+  set slow_count $sqlite_search_count
+  expr {$fast_count < 0.02*$slow_count}
+} {1}
+
+do_execsql_test limit2-110 {
+  CREATE TABLE t2(x,y);
+  INSERT INTO t2(x,y) VALUES('a',1),('a',2),('a',3),('a',4);
+  INSERT INTO t2(x,y) VALUES('b',1),('c',2),('d',3),('e',4);
+  CREATE INDEX t2xy ON t2(x,y);
+}
+set sqlite_search_count 0
+do_execsql_test limit2-110.1 {
+  SELECT a, b, '|' FROM t2, t1 WHERE t2.x='a' AND t1.a=t2.y ORDER BY t1.b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+set fast_count $sqlite_search_count
+set sqlite_search_count 0
+do_execsql_test limit2-110.2 {
+  SELECT a, b, '|' FROM t2, t1 WHERE t2.x='a' AND t1.a=t2.y ORDER BY +t1.b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+set slow_count $sqlite_search_count
+do_test limit2-110.3 {
+  expr {$fast_count < 0.02*$slow_count}
+} {1}
+
+do_execsql_test limit2-120 {
+  DROP INDEX t1ab;
+  CREATE INDEX t1ab ON t1(a,b DESC);
+}
+set sqlite_search_count 0
+do_execsql_test limit2-120.1 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY b DESC LIMIT 5;
+} {5 9999 | 1 1999 | 1 1998 | 1 1997 | 1 1996 |}
+set fast_count $sqlite_search_count
+set sqlite_search_count 0
+do_execsql_test limit2-120.2 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY +b DESC LIMIT 5;
+} {5 9999 | 1 1999 | 1 1998 | 1 1997 | 1 1996 |}
+do_test limit2-120.3 {
+  set slow_count $sqlite_search_count
+  expr {$fast_count < 0.02*$slow_count}
+} {1}
+
+
+
+finish_test

test/misc5.test

@@ -571,6 +571,8 @@ ifcapable subquery&&compound {
 # Overflow the lemon parser stack by providing an overly complex
 # expression.  Make sure that the overflow is detected and reported.
 #
+# This test fails when building with -DYYSTACKDEPTH=0
+#
 do_test misc5-7.1 {
   execsql {CREATE TABLE t1(x)}
   set sql "INSERT INTO t1 VALUES("

test/permutations.test

@@ -746,6 +746,11 @@ test_suite "inmemory_journal" -description {
 
   # WAL mode is different.
   wal* tkt-2d1a5c67d.test backcompat.test e_wal* rowallock.test
+
+  # This test does not work as the "PRAGMA journal_mode = memory"
+  # statement switches the database out of wal mode at inopportune
+  # times.
+  snapshot_fault.test
 }]
 
 ifcapable mem3 {

test/selectA.test

@@ -1434,5 +1434,14 @@ do_catchsql_test 5.4 {
   SELECT * FROM t8 UNION SELECT * FROM t9 ORDER BY a+b COLLATE NOCASE
 } {1 {1st ORDER BY term does not match any column in the result set}}
 
+do_execsql_test 6.1 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(a INTEGER);
+  CREATE TABLE t2(b TEXT);
+  INSERT INTO t2(b) VALUES('12345');
+  SELECT * FROM (SELECT a FROM t1 UNION SELECT b FROM t2) WHERE a=a;
+} {12345}
+
 
 finish_test

test/shell1.test

@@ -915,6 +915,9 @@ do_test shell1-5.0 {
   }
 } {}
 
+# These test cases do not work on MinGW
+if 0 {
+
 # The string used here is the word "test" in Chinese.
 # In UTF-8, it is encoded as: \xE6\xB5\x8B\xE8\xAF\x95
 set test \u6D4B\u8BD5
@@ -951,5 +954,6 @@ do_test shell1-6.1 {
   }
   forcedelete test3.db
 } {}
+}
 
 finish_test

test/snapshot.test

@@ -352,10 +352,6 @@ do_test 6.4 {
   db2 eval { SELECT * FROM x1 }
 } {z zz zzz}
 
-# EVIDENCE-OF: R-55491-50411 A snapshot will fail to open if the
-# database connection D has not previously completed at least one read
-# operation against the database file.
-#
 do_test 6.5 {
   db2 close
   sqlite3 db2 test.db 

test/sort5.test

@@ -78,10 +78,7 @@ execsql { CREATE TABLE t1(x) }
 # each a bit over 100 bytes in size. In total a little more than 1MiB 
 # of data.
 #
-breakpoint
 foreach {tn pgsz cachesz bTemp} {
-  2 1024   1000  1
-
   1 4096   1000  0
   2 1024   1000  1
 
@@ -97,6 +94,12 @@ foreach {tn pgsz cachesz bTemp} {
     PRAGMA cache_size = $cachesz;
   "
 
+  if {[db one {PRAGMA page_size}]!=$pgsz} {
+    # SEE is not able to change page sizes and that messes up the
+    # results that follow.
+    continue
+  }
+
   do_test 2.$tn.1 {
     set ::iTemp 0
     catch { array unset F }
@@ -113,4 +116,3 @@ foreach {tn pgsz cachesz bTemp} {
 }
 
 finish_test
-

test/speedtest1.c

@@ -1216,6 +1216,10 @@ static void displayLinuxIoStats(FILE *out){
 }   
 #endif
 
+#if SQLITE_VERSION_NUMBER<3006018
+#  define sqlite3_sourceid(X) "(before 3.6.18)"
+#endif
+
 int main(int argc, char **argv){
   int doAutovac = 0;            /* True for --autovacuum */
   int cacheSize = 0;            /* Desired cache size.  0 means default */

test/stat.test

@@ -32,6 +32,9 @@ db func a_string a_string
 
 register_dbstat_vtab db
 do_execsql_test stat-0.0 {
+  PRAGMA table_info(dbstat);
+} {/0 name TEXT .* 1 path TEXT .* 9 pgsize INTEGER/}
+do_execsql_test stat-0.1 {
   PRAGMA auto_vacuum = OFF;
   CREATE VIRTUAL TABLE temp.stat USING dbstat;
   SELECT * FROM stat;

test/tclsqlite.test

@@ -635,6 +635,39 @@ do_test tcl-14.2 {
   db one {SELECT x FROM t6 WHERE xCall()!='value'}
 } {}
 
+# Verify that the "exists" and "onecolumn" methods work when
+# a "profile" is registered.
+#
+catch {db close}
+sqlite3 db :memory:
+proc noop-profile {args} {
+  return
+}
+do_test tcl-15.0 {
+  db eval {CREATE TABLE t1(a); INSERT INTO t1 VALUES(1),(2),(3);}
+  db onecolumn {SELECT a FROM t1 WHERE a>2}
+} {3}
+do_test tcl-15.1 {
+  db exists {SELECT a FROM t1 WHERE a>2}
+} {1}
+do_test tcl-15.2 {
+  db exists {SELECT a FROM t1 WHERE a>3}
+} {0}
+db profile noop-profile
+do_test tcl-15.3 {
+  db onecolumn {SELECT a FROM t1 WHERE a>2}
+} {3}
+do_test tcl-15.4 {
+  db exists {SELECT a FROM t1 WHERE a>2}
+} {1}
+do_test tcl-15.5 {
+  db exists {SELECT a FROM t1 WHERE a>3}
+} {0}
+
+
+
+
+
 
 
 finish_test

test/temptable3.test

@@ -0,0 +1,41 @@
+# 2016-05-10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix temptable3
+
+db close
+sqlite3 db {}
+do_execsql_test 1.1 {
+  PRAGMA cache_size = 1;
+  PRAGMA page_size = 1024;
+  PRAGMA auto_vacuum = 2;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES( randomblob(800) );
+  INSERT INTO t1 VALUES( randomblob(800) );
+  CREATE TABLE t2(x);
+  PRAGMA integrity_check;
+} {ok}
+
+db close
+sqlite3 db {}
+do_execsql_test 1.2 {
+  PRAGMA cache_size = 1;
+  PRAGMA auto_vacuum = 2;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x UNIQUE);
+  INSERT INTO t2 VALUES(1), (2), (3);
+  DROP TABLE t1;
+  PRAGMA integrity_check;
+} {ok}
+
+finish_test

test/walcrash4.test

@@ -0,0 +1,84 @@
+# 2010 May 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+ifcapable !wal {finish_test ; return }
+set testprefix walcrash4
+
+#-------------------------------------------------------------------------
+# At one point, if "PRAGMA synchronous=full" is set and the platform
+# does not support POWERSAFE_OVERWRITE, and the last frame written to 
+# the wal file in a transaction is aligned with a sector boundary, the
+# xSync() call was omitted. 
+#
+# The following test verifies that this has been fixed.
+#
+do_execsql_test 1.0 {
+  PRAGMA autovacuum = 0;
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  PRAGMA main.synchronous = full;
+} {wal}
+
+faultsim_save_and_close
+
+# The error message is different on unix and windows
+#
+if {$::tcl_platform(platform)=="windows"} {
+ set msg "child killed: unknown signal"
+} else {
+ set msg "child process exited abnormally"
+}
+
+for {set nExtra 0} {$nExtra < 10} {incr nExtra} {
+  for {set i 0} {$i < 10} {incr i} {
+    do_test 1.nExtra=$nExtra.i=$i.1 {
+      faultsim_restore_and_reopen
+    
+      set fd [open crash.tcl w]
+      puts $fd [subst -nocommands {
+        sqlite3_crash_enable 1
+        sqlite3_test_control_pending_byte $::sqlite_pending_byte
+        sqlite3 db test.db -vfs crash
+        db eval {
+          PRAGMA main.synchronous=FULL;
+          BEGIN;
+          CREATE TABLE t1(x UNIQUE);
+        }
+        for {set e 2} {[set e] < ($nExtra+2)} {incr e} {
+          db eval "CREATE TABLE t[set e] (x)"
+        }
+        db eval {
+          INSERT INTO t1 VALUES( randomblob(170000) );
+          COMMIT;
+        }
+        sqlite3_crash_now
+      }]
+      close $fd
+    
+      set r [catch { exec [info nameofexec] crash.tcl >@stdout } msg]
+      list $r $msg
+    } "1 {$msg}"
+  
+    do_execsql_test 1.nExtra=$nExtra.i=$i.2 { 
+      SELECT count(*) FROM t1;
+      PRAGMA integrity_check;
+    } {1 ok}
+  } 
+}
+
+
+finish_test

test/where3.test

@@ -45,7 +45,7 @@ do_test where3-1.1 {
   }
 } {222 two 2 222 {} {}}
 
-ifcapable explain {
+ifcapable explain&&!cursorhints {
   do_test where3-1.1.1 {
      explain_no_trace {SELECT * FROM t1, t2 LEFT JOIN t3 ON q=x
                         WHERE p=2 AND a=q}
@@ -86,7 +86,7 @@ do_test where3-1.2 {
   }
 } {1 {Value for C1.1} {Value for C2.1} 2 {} {Value for C2.2} 3 {Value for C1.3} {Value for C2.3}}
 
-ifcapable explain {
+ifcapable explain&&!cursorhints {
   do_test where3-1.2.1 {
      explain_no_trace {
        SELECT parent1.parent1key, child1.value, child2.value

tool/dbhash.c

@@ -0,0 +1,506 @@
+/*
+** 2016-06-07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This is a utility program that computes an SHA1 hash on the content
+** of an SQLite database.
+**
+** The hash is computed over just the content of the database.  Free
+** space inside of the database file, and alternative on-disk representations
+** of the same content (ex: UTF8 vs UTF16) do not affect the hash.  So,
+** for example, the database file page size, encoding, and auto_vacuum setting
+** can all be changed without changing the hash.
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <string.h>
+#include <assert.h>
+#include "sqlite3.h"
+
+/* Context for the SHA1 hash */
+typedef struct SHA1Context SHA1Context;
+struct SHA1Context {
+  unsigned int state[5];
+  unsigned int count[2];
+  unsigned char buffer[64];
+};
+
+/*
+** All global variables are gathered into the "g" singleton.
+*/
+struct GlobalVars {
+  const char *zArgv0;       /* Name of program */
+  unsigned fDebug;          /* Debug flags */
+  sqlite3 *db;              /* The database connection */
+  SHA1Context cx;           /* SHA1 hash context */
+} g;
+
+/*
+** Debugging flags
+*/
+#define DEBUG_FULLTRACE   0x00000001   /* Trace hash to stderr */
+
+/******************************************************************************
+** The Hash Engine
+**
+** Modify these routines (and appropriate state fields in global variable 'g')
+** in order to compute a different (better?) hash of the database.
+*/
+/*
+ * blk0() and blk() perform the initial expand.
+ * I got the idea of expanding during the round function from SSLeay
+ *
+ * blk0le() for little-endian and blk0be() for big-endian.
+ */
+#if __GNUC__ && (defined(__i386__) || defined(__x86_64__))
+/*
+ * GCC by itself only generates left rotates.  Use right rotates if
+ * possible to be kinder to dinky implementations with iterative rotate
+ * instructions.
+ */
+#define SHA_ROT(op, x, k) \
+        ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; })
+#define rol(x,k) SHA_ROT("roll", x, k)
+#define ror(x,k) SHA_ROT("rorl", x, k)
+
+#else
+/* Generic C equivalent */
+#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r))
+#define rol(x,k) SHA_ROT(x,k,32-(k))
+#define ror(x,k) SHA_ROT(x,32-(k),k)
+#endif
+
+
+#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \
+    |(rol(block[i],8)&0x00FF00FF))
+#define blk0be(i) block[i]
+#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \
+    ^block[(i+2)&15]^block[i&15],1))
+
+/*
+ * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1
+ *
+ * Rl0() for little-endian and Rb0() for big-endian.  Endianness is
+ * determined at run-time.
+ */
+#define Rl0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define Rb0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R1(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R2(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2);
+#define R3(v,w,x,y,z,i) \
+    z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2);
+#define R4(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2);
+
+/*
+ * Hash a single 512-bit block. This is the core of the algorithm.
+ */
+#define a qq[0]
+#define b qq[1]
+#define c qq[2]
+#define d qq[3]
+#define e qq[4]
+
+void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){
+  unsigned int qq[5]; /* a, b, c, d, e; */
+  static int one = 1;
+  unsigned int block[16];
+  memcpy(block, buffer, 64);
+  memcpy(qq,state,5*sizeof(unsigned int));
+
+  /* Copy g.cx.state[] to working vars */
+  /*
+  a = state[0];
+  b = state[1];
+  c = state[2];
+  d = state[3];
+  e = state[4];
+  */
+
+  /* 4 rounds of 20 operations each. Loop unrolled. */
+  if( 1 == *(unsigned char*)&one ){
+    Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3);
+    Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7);
+    Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11);
+    Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15);
+  }else{
+    Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3);
+    Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7);
+    Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11);
+    Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15);
+  }
+  R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19);
+  R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23);
+  R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27);
+  R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31);
+  R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35);
+  R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39);
+  R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43);
+  R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47);
+  R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51);
+  R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55);
+  R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59);
+  R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63);
+  R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67);
+  R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71);
+  R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75);
+  R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79);
+
+  /* Add the working vars back into context.state[] */
+  state[0] += a;
+  state[1] += b;
+  state[2] += c;
+  state[3] += d;
+  state[4] += e;
+}
+
+
+/* Initialize the SHA1 hash */
+static void hash_init(void){
+  /* SHA1 initialization constants */
+  g.cx.state[0] = 0x67452301;
+  g.cx.state[1] = 0xEFCDAB89;
+  g.cx.state[2] = 0x98BADCFE;
+  g.cx.state[3] = 0x10325476;
+  g.cx.state[4] = 0xC3D2E1F0;
+  g.cx.count[0] = g.cx.count[1] = 0;
+}
+
+/* Add new content to the SHA1 hash */
+static void hash_step(const unsigned char *data,  unsigned int len){
+  unsigned int i, j;
+
+  j = g.cx.count[0];
+  if( (g.cx.count[0] += len << 3) < j ){
+    g.cx.count[1] += (len>>29)+1;
+  }
+  j = (j >> 3) & 63;
+  if( (j + len) > 63 ){
+    (void)memcpy(&g.cx.buffer[j], data, (i = 64-j));
+    SHA1Transform(g.cx.state, g.cx.buffer);
+    for(; i + 63 < len; i += 64){
+      SHA1Transform(g.cx.state, &data[i]);
+    }
+    j = 0;
+  }else{
+    i = 0;
+  }
+  (void)memcpy(&g.cx.buffer[j], &data[i], len - i);
+}
+
+
+/* Add padding and compute and output the message digest. */
+static void hash_finish(const char *zName){
+  unsigned int i;
+  unsigned char finalcount[8];
+  unsigned char digest[20];
+  static const char zEncode[] = "0123456789abcdef";
+  char zOut[41];
+
+  for (i = 0; i < 8; i++){
+    finalcount[i] = (unsigned char)((g.cx.count[(i >= 4 ? 0 : 1)]
+       >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */
+  }
+  hash_step((const unsigned char *)"\200", 1);
+  while ((g.cx.count[0] & 504) != 448){
+    hash_step((const unsigned char *)"\0", 1);
+  }
+  hash_step(finalcount, 8);  /* Should cause a SHA1Transform() */
+  for (i = 0; i < 20; i++){
+    digest[i] = (unsigned char)((g.cx.state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
+  }
+  for(i=0; i<20; i++){
+    zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
+    zOut[i*2+1] = zEncode[digest[i] & 0xf];
+  }
+  zOut[i*2]= 0;
+  printf("%s %s\n", zOut, zName);
+}
+/* End of the hashing logic
+*******************************************************************************/
+  
+/*
+** Print an error resulting from faulting command-line arguments and
+** abort the program.
+*/
+static void cmdlineError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0);
+  exit(1);
+}
+
+/*
+** Print an error message for an error that occurs at runtime, then
+** abort the program.
+*/
+static void runtimeError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n");
+  exit(1);
+}
+
+/*
+** Prepare a new SQL statement.  Print an error and abort if anything
+** goes wrong.
+*/
+static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){
+  char *zSql;
+  int rc;
+  sqlite3_stmt *pStmt;
+
+  zSql = sqlite3_vmprintf(zFormat, ap);
+  if( zSql==0 ) runtimeError("out of memory");
+  rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0);
+  if( rc ){
+    runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db),
+                 zSql);
+  }
+  sqlite3_free(zSql);
+  return pStmt;
+}
+static sqlite3_stmt *db_prepare(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  return pStmt;
+}
+
+/*
+** Compute the hash for all rows of the query formed from the printf-style
+** zFormat and its argument.
+*/
+static void hash_one_query(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;        /* The query defined by zFormat and "..." */
+  int nCol;                   /* Number of columns in the result set */
+  int i;                      /* Loop counter */
+
+  /* Prepare the query defined by zFormat and "..." */
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  nCol = sqlite3_column_count(pStmt);
+
+  /* Compute a hash over the result of the query */
+  while( SQLITE_ROW==sqlite3_step(pStmt) ){
+    for(i=0; i<nCol; i++){
+      switch( sqlite3_column_type(pStmt,i) ){
+        case SQLITE_NULL: {
+          hash_step((const unsigned char*)"0",1);
+          if( g.fDebug & DEBUG_FULLTRACE ) fprintf(stderr, "NULL\n");
+          break;
+        }
+        case SQLITE_INTEGER: {
+          sqlite3_uint64 u;
+          int j;
+          unsigned char x[8];
+          sqlite3_int64 v = sqlite3_column_int64(pStmt,i);
+          memcpy(&u, &v, 8);
+          for(j=7; j>=0; j--){
+            x[j] = u & 0xff;
+            u >>= 8;
+          }
+          hash_step((const unsigned char*)"1",1);
+          hash_step(x,8);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "INT %s\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_FLOAT: {
+          sqlite3_uint64 u;
+          int j;
+          unsigned char x[8];
+          double r = sqlite3_column_double(pStmt,i);
+          memcpy(&u, &r, 8);
+          for(j=7; j>=0; j--){
+            x[j] = u & 0xff;
+            u >>= 8;
+          }
+          hash_step((const unsigned char*)"2",1);
+          hash_step(x,8);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "FLOAT %s\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_TEXT: {
+          int n = sqlite3_column_bytes(pStmt, i);
+          const unsigned char *z = sqlite3_column_text(pStmt, i);
+          hash_step((const unsigned char*)"3", 1);
+          hash_step(z, n);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "TEXT '%s'\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_BLOB: {
+          int n = sqlite3_column_bytes(pStmt, i);
+          const unsigned char *z = sqlite3_column_blob(pStmt, i);
+          hash_step((const unsigned char*)"4", 1);
+          hash_step(z, n);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "BLOB (%d bytes)\n", n);
+          }
+          break;
+        }
+      }
+    }
+  }
+  sqlite3_finalize(pStmt);
+}
+
+
+/*
+** Print sketchy documentation for this utility program
+*/
+static void showHelp(void){
+  printf("Usage: %s [options] FILE ...\n", g.zArgv0);
+  printf(
+"Compute a SHA1 hash on the content of database FILE.  System tables such as\n"
+"sqlite_stat1, sqlite_stat4, and sqlite_sequence are omitted from the hash.\n"
+"Options:\n"
+"   --debug N           Set debugging flags to N (experts only)\n"
+"   --like PATTERN      Only hash tables whose name is LIKE the pattern\n"
+"   --schema-only       Only hash the schema - omit table content\n"
+"   --without-schema    Only hash table content - omit the schema\n"
+  );
+}
+
+int main(int argc, char **argv){
+  const char *zDb = 0;         /* Name of the database currently being hashed */
+  int i;                       /* Loop counter */
+  int rc;                      /* Subroutine return code */
+  char *zErrMsg;               /* Error message when opening database */
+  sqlite3_stmt *pStmt;         /* An SQLite query */
+  const char *zLike = 0;       /* LIKE pattern of tables to hash */
+  int omitSchema = 0;          /* True to compute hash on content only */
+  int omitContent = 0;         /* True to compute hash on schema only */
+  int nFile = 0;               /* Number of input filenames seen */
+
+  g.zArgv0 = argv[0];
+  sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
+  for(i=1; i<argc; i++){
+    const char *z = argv[i];
+    if( z[0]=='-' ){
+      z++;
+      if( z[0]=='-' ) z++;
+      if( strcmp(z,"debug")==0 ){
+        if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
+        g.fDebug = strtol(argv[++i], 0, 0);
+      }else
+      if( strcmp(z,"help")==0 ){
+        showHelp();
+        return 0;
+      }else
+      if( strcmp(z,"like")==0 ){
+        if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
+        if( zLike!=0 ) cmdlineError("only one --like allowed");
+        zLike = argv[++i];
+      }else
+      if( strcmp(z,"schema-only")==0 ){
+        omitContent = 1;
+      }else
+      if( strcmp(z,"without-schema")==0 ){
+        omitSchema = 1;
+      }else
+      {
+        cmdlineError("unknown option: %s", argv[i]);
+      }
+    }else{
+      nFile++;
+      if( nFile<i ) argv[nFile] = argv[i];
+    }
+  }
+  if( nFile==0 ){
+    cmdlineError("no input files specified - nothing to do");
+  }
+  if( omitSchema && omitContent ){
+    cmdlineError("only one of --without-schema and --omit-schema allowed");
+  }
+  if( zLike==0 ) zLike = "%";
+
+  for(i=1; i<=nFile; i++){
+    static const int openFlags = 
+       SQLITE_OPEN_READWRITE |     /* Read/write so hot journals can recover */
+       SQLITE_OPEN_URI
+    ;
+    zDb = argv[i];
+    rc = sqlite3_open_v2(zDb, &g.db, openFlags, 0);
+    if( rc ){
+      fprintf(stderr, "cannot open database file '%s'\n", zDb);
+      continue;
+    }
+    rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_master", 0, 0, &zErrMsg);
+    if( rc || zErrMsg ){
+      sqlite3_close(g.db);
+      g.db = 0;
+      fprintf(stderr, "'%s' is not a valid SQLite database\n", zDb);
+      continue;
+    }
+
+    /* Start the hash */
+    hash_init();
+  
+    /* Hash table content */
+    if( !omitContent ){
+      pStmt = db_prepare(
+        "SELECT name FROM sqlite_master\n"
+        " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+        "   AND name NOT LIKE 'sqlite_%%'\n"
+        "   AND name LIKE '%q'\n"
+        " ORDER BY name COLLATE nocase;\n",
+        zLike
+      );
+      while( SQLITE_ROW==sqlite3_step(pStmt) ){
+        /* We want rows of the table to be hashed in PRIMARY KEY order.
+        ** Technically, an ORDER BY clause is required to guarantee that
+        ** order.  However, though not guaranteed by the documentation, every
+        ** historical version of SQLite has always output rows in PRIMARY KEY
+        ** order when there is no WHERE or GROUP BY clause, so the ORDER BY
+        ** can be safely omitted. */
+        hash_one_query("SELECT * FROM \"%w\"", sqlite3_column_text(pStmt,0));
+      }
+      sqlite3_finalize(pStmt);
+    }
+  
+    /* Hash the database schema */
+    if( !omitSchema ){
+      hash_one_query(
+         "SELECT type, name, tbl_name, sql FROM sqlite_master\n"
+         " WHERE tbl_name LIKE '%q'\n"
+         " ORDER BY name COLLATE nocase;\n",
+         zLike
+      );
+    }
+  
+    /* Finish and output the hash and close the database connection. */
+    hash_finish(zDb);
+    sqlite3_close(g.db);
+  }
+  return 0;
+}

tool/lemon.c

@@ -288,10 +288,13 @@ struct rule {
   const char *code;        /* The code executed when this rule is reduced */
   const char *codePrefix;  /* Setup code before code[] above */
   const char *codeSuffix;  /* Breakdown code after code[] above */
+  int noCode;              /* True if this rule has no associated C code */
+  int codeEmitted;         /* True if the code has been emitted already */
   struct symbol *precsym;  /* Precedence symbol for this rule */
   int index;               /* An index number for this rule */
   int iRule;               /* Rule number as used in the generated tables */
   Boolean canReduce;       /* True if this rule is ever reduced */
+  Boolean doesReduce;      /* Reduce actions occur after optimization */
   struct rule *nextlhs;    /* Next rule with the same LHS */
   struct rule *next;       /* Next rule in the global list */
 };
@@ -339,6 +342,7 @@ struct action {
     struct state *stp;     /* The new state, if a shift */
     struct rule *rp;       /* The rule, if a reduce */
   } x;
+  struct symbol *spOpt;    /* SHIFTREDUCE optimization to this symbol */
   struct action *next;     /* Next action for this state */
   struct action *collide;  /* Next action with the same hash */
 };
@@ -349,7 +353,7 @@ struct state {
   struct config *bp;       /* The basis configurations for this state */
   struct config *cfp;      /* All configurations in this set */
   int statenum;            /* Sequential number for this state */
-  struct action *ap;       /* Array of actions for this state */
+  struct action *ap;       /* List of actions for this state */
   int nTknAct, nNtAct;     /* Number of actions on terminals and nonterminals */
   int iTknOfst, iNtOfst;   /* yy_action[] offset for terminals and nonterms */
   int iDfltReduce;         /* Default action is to REDUCE by this rule */
@@ -530,6 +534,7 @@ void Action_add(
   *app = newaction;
   newaction->type = type;
   newaction->sp = sp;
+  newaction->spOpt = 0;
   if( type==SHIFT ){
     newaction->x.stp = (struct state *)arg;
   }else{
@@ -1499,7 +1504,7 @@ static void handle_T_option(char *z){
   lemon_strcpy(user_templatename, z);
 }
 
-/* Merge together to lists of rules order by rule.iRule */
+/* Merge together to lists of rules ordered by rule.iRule */
 static struct rule *Rule_merge(struct rule *pA, struct rule *pB){
   struct rule *pFirst = 0;
   struct rule **ppPrev = &pFirst;
@@ -1642,7 +1647,10 @@ int main(int argc, char **argv)
   for(i=1; ISUPPER(lem.symbols[i]->name[0]); i++);
   lem.nterminal = i;
 
-  /* Assign sequential rule numbers */
+  /* Assign sequential rule numbers.  Start with 0.  Put rules that have no
+  ** reduce action C-code associated with them last, so that the switch()
+  ** statement that selects reduction actions will have a smaller jump table.
+  */
   for(i=0, rp=lem.rule; rp; rp=rp->next){
     rp->iRule = rp->code ? i++ : -1;
   }
@@ -2211,6 +2219,7 @@ to follow the previous rule.");
         }else{
           psp->prevrule->line = psp->tokenlineno;
           psp->prevrule->code = &x[1];
+          psp->prevrule->noCode = 0;
         }
       }else if( x[0]=='[' ){
         psp->state = PRECEDENCE_MARK_1;
@@ -2317,6 +2326,7 @@ to follow the previous rule.");
           rp->lhsalias = psp->lhsalias;
           rp->nrhs = psp->nrhs;
           rp->code = 0;
+          rp->noCode = 1;
           rp->precsym = 0;
           rp->index = psp->gp->nrule++;
           rp->nextlhs = rp->lhs->rule;
@@ -3160,6 +3170,9 @@ int PrintAction(
       result = 0;
       break;
   }
+  if( result && ap->spOpt ){
+    fprintf(fp,"  /* because %s==%s */", ap->sp->name, ap->spOpt->name);
+  }
   return result;
 }
 
@@ -3527,9 +3540,8 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2){
 }
 
 /*
-** zCode is a string that is the action associated with a rule.  Expand
-** the symbols in this string so that the refer to elements of the parser
-** stack.
+** Write and transform the rp->code string so that symbols are expanded.
+** Populate the rp->codePrefix and rp->codeSuffix strings, as appropriate.
 **
 ** Return 1 if the expanded code requires that "yylhsminor" local variable
 ** to be defined.
@@ -3553,6 +3565,9 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
     static char newlinestr[2] = { '\n', '\0' };
     rp->code = newlinestr;
     rp->line = rp->ruleline;
+    rp->noCode = 1;
+  }else{
+    rp->noCode = 0;
   }
 
 
@@ -3568,6 +3583,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
       append_str("  yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0,
                  rp->rhs[0]->index,1-rp->nrhs);
       rp->codePrefix = Strsafe(append_str(0,0,0,0));
+      rp->noCode = 0;
     }
   }else if( rp->lhsalias==0 ){
     /* There is no LHS value symbol. */
@@ -3715,7 +3731,10 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
 
   /* Suffix code generation complete */
   cp = append_str(0,0,0,0);
-  if( cp && cp[0] ) rp->codeSuffix = Strsafe(cp);
+  if( cp && cp[0] ){
+    rp->codeSuffix = Strsafe(cp);
+    rp->noCode = 0;
+  }
 
   return rc;
 }
@@ -4134,6 +4153,19 @@ void ReportTable(
   }
   free(ax);
 
+  /* Mark rules that are actually used for reduce actions after all
+  ** optimizations have been applied
+  */
+  for(rp=lemp->rule; rp; rp=rp->next) rp->doesReduce = LEMON_FALSE;
+  for(i=0; i<lemp->nxstate; i++){
+    struct action *ap;
+    for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){
+      if( ap->type==REDUCE || ap->type==SHIFTREDUCE ){
+        ap->x.rp->doesReduce = i;
+      }
+    }
+  }
+
   /* Finish rendering the constants now that the action table has
   ** been computed */
   fprintf(out,"#define YYNSTATE             %d\n",lemp->nxstate);  lineno++;
@@ -4396,13 +4428,9 @@ void ReportTable(
   /* First output rules other than the default: rule */
   for(rp=lemp->rule; rp; rp=rp->next){
     struct rule *rp2;               /* Other rules with the same action */
-    if( rp->code==0 ) continue;
-    if( rp->code[0]=='\n'
-     && rp->code[1]==0
-     && rp->codePrefix==0
-     && rp->codeSuffix==0
-    ){
-      /* No actions, so this will be part of the "default:" rule */
+    if( rp->codeEmitted ) continue;
+    if( rp->noCode ){
+      /* No C code actions, so this will be part of the "default:" rule */
       continue;
     }
     fprintf(out,"      case %d: /* ", rp->iRule);
@@ -4414,24 +4442,27 @@ void ReportTable(
         fprintf(out,"      case %d: /* ", rp2->iRule);
         writeRuleText(out, rp2);
         fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++;
-        rp2->code = 0;
+        rp2->codeEmitted = 1;
       }
     }
     emit_code(out,rp,lemp,&lineno);
     fprintf(out,"        break;\n"); lineno++;
-    rp->code = 0;
+    rp->codeEmitted = 1;
   }
   /* Finally, output the default: rule.  We choose as the default: all
   ** empty actions. */
   fprintf(out,"      default:\n"); lineno++;
   for(rp=lemp->rule; rp; rp=rp->next){
-    if( rp->code==0 ) continue;
-    assert( rp->code[0]=='\n' && rp->code[1]==0 );
-    assert( rp->codePrefix==0 );
-    assert( rp->codeSuffix==0 );
+    if( rp->codeEmitted ) continue;
+    assert( rp->noCode );
     fprintf(out,"      /* (%d) ", rp->iRule);
     writeRuleText(out, rp);
-    fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++;
+    if( rp->doesReduce ){
+      fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++;
+    }else{
+      fprintf(out, " (OPTIMIZED OUT) */ assert(yyruleno!=%d);\n",
+              rp->iRule); lineno++;
+    }
   }
   fprintf(out,"        break;\n"); lineno++;
   tplt_xfer(lemp->name,in,out,&lineno);
@@ -4502,7 +4533,7 @@ void ReportHeader(struct lemon *lemp)
 void CompressTables(struct lemon *lemp)
 {
   struct state *stp;
-  struct action *ap, *ap2;
+  struct action *ap, *ap2, *nextap;
   struct rule *rp, *rp2, *rbest;
   int nbest, n;
   int i;
@@ -4579,6 +4610,36 @@ void CompressTables(struct lemon *lemp)
       }
     }
   }
+
+  /* If a SHIFTREDUCE action specifies a rule that has a single RHS term
+  ** (meaning that the SHIFTREDUCE will land back in the state where it
+  ** started) and if there is no C-code associated with the reduce action,
+  ** then we can go ahead and convert the action to be the same as the
+  ** action for the RHS of the rule.
+  */
+  for(i=0; i<lemp->nstate; i++){
+    stp = lemp->sorted[i];
+    for(ap=stp->ap; ap; ap=nextap){
+      nextap = ap->next;
+      if( ap->type!=SHIFTREDUCE ) continue;
+      rp = ap->x.rp;
+      if( rp->noCode==0 ) continue;
+      if( rp->nrhs!=1 ) continue;
+#if 1
+      /* Only apply this optimization to non-terminals.  It would be OK to
+      ** apply it to terminal symbols too, but that makes the parser tables
+      ** larger. */
+      if( ap->sp->index<lemp->nterminal ) continue;
+#endif
+      /* If we reach this point, it means the optimization can be applied */
+      nextap = ap;
+      for(ap2=stp->ap; ap2 && (ap2==ap || ap2->sp!=rp->lhs); ap2=ap2->next){}
+      assert( ap2!=0 );
+      ap->spOpt = ap2->sp;
+      ap->type = ap2->type;
+      ap->x = ap2->x;
+    }
+  }
 }
 
 

tool/lempar.c

@@ -203,9 +203,9 @@ typedef struct yyStackEntry yyStackEntry;
 /* The state of the parser is completely contained in an instance of
 ** the following structure */
 struct yyParser {
-  int yyidx;                    /* Index of top element in stack */
+  yyStackEntry *yytos;          /* Pointer to top element of the stack */
 #ifdef YYTRACKMAXSTACKDEPTH
-  int yyidxMax;                 /* Maximum value of yyidx */
+  int yyhwm;                    /* High-water mark of the stack */
 #endif
 #ifndef YYNOERRORRECOVERY
   int yyerrcnt;                 /* Shifts left before out of the error */
@@ -214,6 +214,7 @@ struct yyParser {
 #if YYSTACKDEPTH<=0
   int yystksz;                  /* Current side of the stack */
   yyStackEntry *yystack;        /* The parser's stack */
+  yyStackEntry yystk0;          /* First stack entry */
 #else
   yyStackEntry yystack[YYSTACKDEPTH];  /* The parser's stack */
 #endif
@@ -271,24 +272,34 @@ static const char *const yyRuleName[] = {
 
 #if YYSTACKDEPTH<=0
 /*
-** Try to increase the size of the parser stack.
+** Try to increase the size of the parser stack.  Return the number
+** of errors.  Return 0 on success.
 */
-static void yyGrowStack(yyParser *p){
+static int yyGrowStack(yyParser *p){
   int newSize;
+  int idx;
   yyStackEntry *pNew;
 
   newSize = p->yystksz*2 + 100;
-  pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
+  idx = p->yytos ? (int)(p->yytos - p->yystack) : 0;
+  if( p->yystack==&p->yystk0 ){
+    pNew = malloc(newSize*sizeof(pNew[0]));
+    if( pNew ) pNew[0] = p->yystk0;
+  }else{
+    pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
+  }
   if( pNew ){
     p->yystack = pNew;
-    p->yystksz = newSize;
+    p->yytos = &p->yystack[idx];
 #ifndef NDEBUG
     if( yyTraceFILE ){
-      fprintf(yyTraceFILE,"%sStack grows to %d entries!\n",
-              yyTracePrompt, p->yystksz);
+      fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n",
+              yyTracePrompt, p->yystksz, newSize);
     }
 #endif
+    p->yystksz = newSize;
   }
+  return pNew==0; 
 }
 #endif
 
@@ -317,15 +328,24 @@ void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){
   yyParser *pParser;
   pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
   if( pParser ){
-    pParser->yyidx = -1;
 #ifdef YYTRACKMAXSTACKDEPTH
-    pParser->yyidxMax = 0;
+    pParser->yyhwm = 0;
 #endif
 #if YYSTACKDEPTH<=0
+    pParser->yytos = NULL;
     pParser->yystack = NULL;
     pParser->yystksz = 0;
-    yyGrowStack(pParser);
+    if( yyGrowStack(pParser) ){
+      pParser->yystack = &pParser->yystk0;
+      pParser->yystksz = 1;
+    }
 #endif
+#ifndef YYNOERRORRECOVERY
+    pParser->yyerrcnt = -1;
+#endif
+    pParser->yytos = pParser->yystack;
+    pParser->yystack[0].stateno = 0;
+    pParser->yystack[0].major = 0;
   }
   return pParser;
 }
@@ -369,8 +389,8 @@ static void yy_destructor(
 */
 static void yy_pop_parser_stack(yyParser *pParser){
   yyStackEntry *yytos;
-  assert( pParser->yyidx>=0 );
-  yytos = &pParser->yystack[pParser->yyidx--];
+  assert( pParser->yytos!=0 );
+  yytos = pParser->yytos--;
 #ifndef NDEBUG
   if( yyTraceFILE ){
     fprintf(yyTraceFILE,"%sPopping %s\n",
@@ -397,9 +417,9 @@ void ParseFree(
 #ifndef YYPARSEFREENEVERNULL
   if( pParser==0 ) return;
 #endif
-  while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser);
+  while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser);
 #if YYSTACKDEPTH<=0
-  free(pParser->yystack);
+  if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack);
 #endif
   (*freeProc)((void*)pParser);
 }
@@ -410,7 +430,7 @@ void ParseFree(
 #ifdef YYTRACKMAXSTACKDEPTH
 int ParseStackPeak(void *p){
   yyParser *pParser = (yyParser*)p;
-  return pParser->yyidxMax;
+  return pParser->yyhwm;
 }
 #endif
 
@@ -423,7 +443,7 @@ static unsigned int yy_find_shift_action(
   YYCODETYPE iLookAhead     /* The look-ahead token */
 ){
   int i;
-  int stateno = pParser->yystack[pParser->yyidx].stateno;
+  int stateno = pParser->yytos->stateno;
  
   if( stateno>=YY_MIN_REDUCE ) return stateno;
   assert( stateno <= YY_SHIFT_COUNT );
@@ -516,13 +536,13 @@ static int yy_find_reduce_action(
 */
 static void yyStackOverflow(yyParser *yypParser){
    ParseARG_FETCH;
-   yypParser->yyidx--;
+   yypParser->yytos--;
 #ifndef NDEBUG
    if( yyTraceFILE ){
      fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt);
    }
 #endif
-   while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+   while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser);
    /* Here code is inserted which will execute if the parser
    ** stack every overflows */
 /******** Begin %stack_overflow code ******************************************/
@@ -539,11 +559,11 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState){
   if( yyTraceFILE ){
     if( yyNewState<YYNSTATE ){
       fprintf(yyTraceFILE,"%sShift '%s', go to state %d\n",
-         yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major],
+         yyTracePrompt,yyTokenName[yypParser->yytos->major],
          yyNewState);
     }else{
       fprintf(yyTraceFILE,"%sShift '%s'\n",
-         yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major]);
+         yyTracePrompt,yyTokenName[yypParser->yytos->major]);
     }
   }
 }
@@ -561,27 +581,30 @@ static void yy_shift(
   ParseTOKENTYPE yyMinor        /* The minor token to shift in */
 ){
   yyStackEntry *yytos;
-  yypParser->yyidx++;
+  yypParser->yytos++;
 #ifdef YYTRACKMAXSTACKDEPTH
-  if( yypParser->yyidx>yypParser->yyidxMax ){
-    yypParser->yyidxMax = yypParser->yyidx;
+  if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
+    yypParser->yyhwm++;
+    assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) );
   }
 #endif
 #if YYSTACKDEPTH>0 
-  if( yypParser->yyidx>=YYSTACKDEPTH ){
+  if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH] ){
     yyStackOverflow(yypParser);
     return;
   }
 #else
-  if( yypParser->yyidx>=yypParser->yystksz ){
-    yyGrowStack(yypParser);
-    if( yypParser->yyidx>=yypParser->yystksz ){
+  if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){
+    if( yyGrowStack(yypParser) ){
       yyStackOverflow(yypParser);
       return;
     }
   }
 #endif
-  yytos = &yypParser->yystack[yypParser->yyidx];
+  if( yyNewState > YY_MAX_SHIFT ){
+    yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+  }
+  yytos = yypParser->yytos;
   yytos->stateno = (YYACTIONTYPE)yyNewState;
   yytos->major = (YYCODETYPE)yyMajor;
   yytos->minor.yy0 = yyMinor;
@@ -613,7 +636,7 @@ static void yy_reduce(
   yyStackEntry *yymsp;            /* The top of the parser's stack */
   int yysize;                     /* Amount to pop the stack */
   ParseARG_FETCH;
-  yymsp = &yypParser->yystack[yypParser->yyidx];
+  yymsp = yypParser->yytos;
 #ifndef NDEBUG
   if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
     yysize = yyRuleInfo[yyruleno].nrhs;
@@ -627,22 +650,23 @@ static void yy_reduce(
   ** enough on the stack to push the LHS value */
   if( yyRuleInfo[yyruleno].nrhs==0 ){
 #ifdef YYTRACKMAXSTACKDEPTH
-    if( yypParser->yyidx>yypParser->yyidxMax ){
-      yypParser->yyidxMax = yypParser->yyidx;
+    if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
+      yypParser->yyhwm++;
+      assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
     }
 #endif
 #if YYSTACKDEPTH>0 
-    if( yypParser->yyidx>=YYSTACKDEPTH-1 ){
+    if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH-1] ){
       yyStackOverflow(yypParser);
       return;
     }
 #else
-    if( yypParser->yyidx>=yypParser->yystksz-1 ){
-      yyGrowStack(yypParser);
-      if( yypParser->yyidx>=yypParser->yystksz-1 ){
+    if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){
+      if( yyGrowStack(yypParser) ){
         yyStackOverflow(yypParser);
         return;
       }
+      yymsp = yypParser->yytos;
     }
 #endif
   }
@@ -665,15 +689,17 @@ static void yy_reduce(
   yysize = yyRuleInfo[yyruleno].nrhs;
   yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto);
   if( yyact <= YY_MAX_SHIFTREDUCE ){
-    if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
-    yypParser->yyidx -= yysize - 1;
+    if( yyact>YY_MAX_SHIFT ){
+      yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+    }
     yymsp -= yysize-1;
+    yypParser->yytos = yymsp;
     yymsp->stateno = (YYACTIONTYPE)yyact;
     yymsp->major = (YYCODETYPE)yygoto;
     yyTraceShift(yypParser, yyact);
   }else{
     assert( yyact == YY_ACCEPT_ACTION );
-    yypParser->yyidx -= yysize;
+    yypParser->yytos -= yysize;
     yy_accept(yypParser);
   }
 }
@@ -691,7 +717,7 @@ static void yy_parse_failed(
     fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
   }
 #endif
-  while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+  while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser);
   /* Here code is inserted which will be executed whenever the
   ** parser fails */
 /************ Begin %parse_failure code ***************************************/
@@ -729,7 +755,7 @@ static void yy_accept(
     fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
   }
 #endif
-  while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+  assert( yypParser->yytos==yypParser->yystack );
   /* Here code is inserted which will be executed whenever the
   ** parser accepts */
 /*********** Begin %parse_accept code *****************************************/
@@ -775,26 +801,7 @@ void Parse(
 
   /* (re)initialize the parser, if necessary */
   yypParser = (yyParser*)yyp;
-  if( yypParser->yyidx<0 ){
-#if YYSTACKDEPTH<=0
-    if( yypParser->yystksz <=0 ){
-      yyStackOverflow(yypParser);
-      return;
-    }
-#endif
-    yypParser->yyidx = 0;
-#ifndef YYNOERRORRECOVERY
-    yypParser->yyerrcnt = -1;
-#endif
-    yypParser->yystack[0].stateno = 0;
-    yypParser->yystack[0].major = 0;
-#ifndef NDEBUG
-    if( yyTraceFILE ){
-      fprintf(yyTraceFILE,"%sInitialize. Empty stack. State 0\n",
-              yyTracePrompt);
-    }
-#endif
-  }
+  assert( yypParser->yytos!=0 );
 #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
   yyendofinput = (yymajor==0);
 #endif
@@ -809,7 +816,6 @@ void Parse(
   do{
     yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
     if( yyact <= YY_MAX_SHIFTREDUCE ){
-      if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
       yy_shift(yypParser,yyact,yymajor,yyminor);
 #ifndef YYNOERRORRECOVERY
       yypParser->yyerrcnt--;
@@ -851,7 +857,7 @@ void Parse(
       if( yypParser->yyerrcnt<0 ){
         yy_syntax_error(yypParser,yymajor,yyminor);
       }
-      yymx = yypParser->yystack[yypParser->yyidx].major;
+      yymx = yypParser->yytos->major;
       if( yymx==YYERRORSYMBOL || yyerrorhit ){
 #ifndef NDEBUG
         if( yyTraceFILE ){
@@ -862,16 +868,15 @@ void Parse(
         yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion);
         yymajor = YYNOCODE;
       }else{
-        while(
-          yypParser->yyidx >= 0 &&
-          yymx != YYERRORSYMBOL &&
-          (yyact = yy_find_reduce_action(
-                        yypParser->yystack[yypParser->yyidx].stateno,
+        while( yypParser->yytos >= &yypParser->yystack
+            && yymx != YYERRORSYMBOL
+            && (yyact = yy_find_reduce_action(
+                        yypParser->yytos->stateno,
                         YYERRORSYMBOL)) >= YY_MIN_REDUCE
         ){
           yy_pop_parser_stack(yypParser);
         }
-        if( yypParser->yyidx < 0 || yymajor==0 ){
+        if( yypParser->yytos < yypParser->yystack || yymajor==0 ){
           yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
           yy_parse_failed(yypParser);
           yymajor = YYNOCODE;
@@ -914,14 +919,16 @@ void Parse(
       yymajor = YYNOCODE;
 #endif
     }
-  }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 );
+  }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack );
 #ifndef NDEBUG
   if( yyTraceFILE ){
-    int i;
+    yyStackEntry *i;
+    char cDiv = '[';
     fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt);
-    for(i=1; i<=yypParser->yyidx; i++)
-      fprintf(yyTraceFILE,"%c%s", i==1 ? '[' : ' ', 
-              yyTokenName[yypParser->yystack[i].major]);
+    for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){
+      fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]);
+      cDiv = ' ';
+    }
     fprintf(yyTraceFILE,"]\n");
   }
 #endif

tool/libvers.c

@@ -0,0 +1,15 @@
+/*
+** Compile this program against an SQLite library of unknown version
+** and then run this program, and it will print out the SQLite version
+** information.
+*/
+#include <stdio.h>
+
+extern const char *sqlite3_libversion(void);
+extern const char *sqlite3_sourceid(void);
+
+int main(int argc, char **argv){
+  printf("SQLite version %s\n", sqlite3_libversion());
+  printf("SQLite source  %s\n", sqlite3_sourceid());
+  return 0;
+}

tool/opcodesum.tcl

@@ -0,0 +1,34 @@
+#!/usr/bin/tclsh
+#
+# Run this script, redirecting input from cachegrind output, to compute the
+# number of CPU cycles used by each VDBE opcode.
+#
+# The cachegrind output should be configured so that it reports a single
+# column of Ir at the left margin. Ex:
+#
+#    cg_annotation --show=Ir --auto=yes cachegrind.out.* | tclsh opcodesum.tcl
+#
+set currentop x
+set ncycle(x) 0
+while {![eof stdin]} {
+  set line [string map {\173 x \175 x \042 x} [gets stdin]]
+  if {[regexp {  \.  case OP_.*:} $line]} {
+    regexp {OP_(.+):} $line all currentop
+    set ncycle($currentop) 0
+  } elseif {[lindex $line 1]=="default:"
+            && [regexp {really OP_Noop and OP_Explain} $line]} {
+    break
+  } elseif {[lindex $line 0]!="."} {
+    regsub -all {[^0-9]} [lindex $line 0] {} n
+    if {$n!=""} {incr ncycle($currentop) $n}
+  }
+}
+unset ncycle(x)
+set results {}
+foreach op [lsort [array names ncycle]] {
+  if {$ncycle($op)==0} continue
+  lappend results [list $ncycle($op) $op]
+}
+foreach entry [lsort -index 0 -int -decr $results] {
+  puts [format {%-16s %10d} [lindex $entry 1] [lindex $entry 0]]
+}

tool/speed-check.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+#
+# This is a template for a script used for day-to-day size and 
+# performance monitoring of SQLite.  Typical usage:
+#
+#     sh run-speed-test.sh trunk  #  Baseline measurement of trunk
+#     sh run-speed-test.sh x1     # Measure some experimental change
+#     fossil test-diff --tk cout-trunk.txt cout-x1.txt   # View chanages
+#
+# There are multiple output files, all with a base name given by
+# the first argument:
+#
+#     summary-$BASE.txt           # Copy of standard output
+#     cout-$BASE.txt              # cachegrind output
+#     explain-$BASE.txt           # EXPLAIN listings (only with --explain)
+#
+if test "$1" = ""
+then
+  echo "Usage: $0 OUTPUTFILE [OPTIONS]"
+  exit
+fi
+NAME=$1
+shift
+CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
+SPEEDTEST_OPTS="--shrink-memory --reprepare --heap 10000000 64"
+SIZE=5
+doExplain=0
+doCachegrind=1
+while test "$1" != ""; do
+  case $1 in
+    --reprepare)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --autovacuum)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --utf16be)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --stats)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --without-rowid)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --nomemstat)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --temp)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6"
+        ;;
+    --wal)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
+        ;;
+    --size)
+        shift; SIZE=$1
+        ;;
+    --explain)
+        doExplain=1
+        ;;
+    --vdbeprofile)
+        rm -f vdbe_profile.out
+        CC_OPTS="$CC_OPTS -DVDBE_PROFILE"
+        doCachegrind=0
+        ;;
+    --heap)
+        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5"
+        shift;
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64"
+        ;;
+    *)
+        CC_OPTS="$CC_OPTS $1"
+        ;;
+  esac
+  shift
+done
+SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE"
+echo "NAME           = $NAME" | tee summary-$NAME.txt
+echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
+echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
+rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
+gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
+size sqlite3.o | tee -a summary-$NAME.txt
+if test $doExplain -eq 1; then
+  gcc -g -Os -Wall -I. $CC_OPTS \
+     -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
+    ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
+fi
+SRC=./speedtest1.c
+gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
+ls -l speedtest1 | tee -a summary-$NAME.txt
+if test $doCachegrind -eq 1; then
+  valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
+      $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
+else
+  ./speedtest1 speedtest1.db $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
+fi
+size sqlite3.o | tee -a summary-$NAME.txt
+wc sqlite3.c
+if test $doCachegrind -eq 1; then
+  cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
+fi
+if test $doExplain -eq 1; then
+  ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
+fi

tool/sqldiff.c

@@ -33,6 +33,7 @@ struct GlobalVars {
   const char *zArgv0;       /* Name of program */
   int bSchemaOnly;          /* Only show schema differences */
   int bSchemaPK;            /* Use the schema-defined PK, not the true PK */
+  int bHandleVtab;          /* Handle fts3, fts4, fts5 and rtree vtabs */
   unsigned fDebug;          /* Debug flags */
   sqlite3 *db;              /* The database connection */
 } g;
@@ -1735,6 +1736,144 @@ end_changeset_one_table:
   sqlite3_free(zId);
 }
 
+/*
+** Extract the next SQL keyword or quoted string from buffer zIn and copy it
+** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
+** Return a pointer to the character within zIn immediately following 
+** the token or quoted string just extracted.
+*/
+const char *gobble_token(const char *zIn, char *zBuf, int nBuf){
+  const char *p = zIn;
+  char *pOut = zBuf;
+  char *pEnd = &pOut[nBuf-1];
+  char q = 0;                     /* quote character, if any */
+
+  if( p==0 ) return 0;
+  while( *p==' ' ) p++;
+  switch( *p ){
+    case '"': q = '"'; break;
+    case '\'': q = '\''; break;
+    case '`': q = '`'; break;
+    case '[': q = ']'; break;
+  }
+
+  if( q ){
+    p++;
+    while( *p && pOut<pEnd ){
+      if( *p==q ){
+        p++;
+        if( *p!=q ) break;
+      }
+      if( pOut<pEnd ) *pOut++ = *p;
+      p++;
+    }
+  }else{
+    while( *p && *p!=' ' && *p!='(' ){
+      if( pOut<pEnd ) *pOut++ = *p;
+      p++;
+    }
+  }
+
+  *pOut = '\0';
+  return p;
+}
+
+/*
+** This function is the implementation of SQL scalar function "module_name":
+**
+**   module_name(SQL)
+**
+** The only argument should be an SQL statement of the type that may appear
+** in the sqlite_master table. If the statement is a "CREATE VIRTUAL TABLE"
+** statement, then the value returned is the name of the module that it
+** uses. Otherwise, if the statement is not a CVT, NULL is returned.
+*/
+static void module_name_func(
+  sqlite3_context *pCtx, 
+  int nVal, sqlite3_value **apVal
+){
+  const char *zSql;
+  char zToken[32];
+
+  assert( nVal==1 );
+  zSql = (const char*)sqlite3_value_text(apVal[0]);
+
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "create") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "virtual") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "table") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "using") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  
+  sqlite3_result_text(pCtx, zToken, -1, SQLITE_TRANSIENT);
+}
+
+/*
+** Return the text of an SQL statement that itself returns the list of
+** tables to process within the database.
+*/
+const char *all_tables_sql(){
+  if( g.bHandleVtab ){
+    int rc;
+  
+    rc = sqlite3_exec(g.db, 
+        "CREATE TEMP TABLE tblmap(module, postfix);"
+        "INSERT INTO temp.tblmap VALUES"
+        "('fts3', '_content'), ('fts3', '_segments'), ('fts3', '_segdir'),"
+  
+        "('fts4', '_content'), ('fts4', '_segments'), ('fts4', '_segdir'),"
+        "('fts4', '_docsize'), ('fts4', '_stat'),"
+  
+        "('fts5', '_data'), ('fts5', '_idx'), ('fts5', '_content'),"
+        "('fts5', '_docsize'), ('fts5', '_config'),"
+  
+        "('rtree', '_node'), ('rtree', '_rowid'), ('rtree', '_parent');"
+        , 0, 0, 0
+    );
+    assert( rc==SQLITE_OK );
+  
+    rc = sqlite3_create_function(
+        g.db, "module_name", 1, SQLITE_UTF8, 0, module_name_func, 0, 0
+    );
+    assert( rc==SQLITE_OK );
+  
+    return 
+      "SELECT name FROM main.sqlite_master\n"
+      " WHERE type='table' AND (\n"
+      "    module_name(sql) IS NULL OR \n"
+      "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
+      " ) AND name NOT IN (\n"
+      "  SELECT a.name || b.postfix \n"
+        "FROM main.sqlite_master AS a, temp.tblmap AS b \n"
+        "WHERE module_name(a.sql) = b.module\n" 
+      " )\n"
+      "UNION \n"
+      "SELECT name FROM aux.sqlite_master\n"
+      " WHERE type='table' AND (\n"
+      "    module_name(sql) IS NULL OR \n"
+      "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
+      " ) AND name NOT IN (\n"
+      "  SELECT a.name || b.postfix \n"
+        "FROM aux.sqlite_master AS a, temp.tblmap AS b \n"
+        "WHERE module_name(a.sql) = b.module\n" 
+      " )\n"
+      " ORDER BY name";
+  }else{
+    return
+      "SELECT name FROM main.sqlite_master\n"
+      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+      " UNION\n"
+      "SELECT name FROM aux.sqlite_master\n"
+      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+      " ORDER BY name";
+  }
+}
+
 /*
 ** Print sketchy documentation for this utility program
 */
@@ -1751,6 +1890,7 @@ static void showHelp(void){
 "  --summary             Show only a summary of the differences\n"
 "  --table TAB           Show only differences in table TAB\n"
 "  --transaction         Show SQL output inside a transaction\n"
+"  --vtab                Handle fts3, fts4, fts5 and rtree tables\n"
   );
 }
 
@@ -1821,6 +1961,9 @@ int main(int argc, char **argv){
       if( strcmp(z,"transaction")==0 ){
         useTransaction = 1;
       }else
+      if( strcmp(z,"vtab")==0 ){
+        g.bHandleVtab = 1;
+      }else
       {
         cmdlineError("unknown option: %s", argv[i]);
       }
@@ -1875,14 +2018,7 @@ int main(int argc, char **argv){
     xDiff(zTab, out);
   }else{
     /* Handle tables one by one */
-    pStmt = db_prepare(
-      "SELECT name FROM main.sqlite_master\n"
-      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
-      " UNION\n"
-      "SELECT name FROM aux.sqlite_master\n"
-      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
-      " ORDER BY name"
-    );
+    pStmt = db_prepare( all_tables_sql() );
     while( SQLITE_ROW==sqlite3_step(pStmt) ){
       xDiff((const char*)sqlite3_column_text(pStmt,0), out);
     }