Add the sqlite3rbu_state() API. Used to determine the current state (creating OAL, ready to move OAL, incremental-checkpoint, finished or error) of an RBU operation.

FossilOrigin-Name: 92e7df0ff5c4c118c63d92a767dc82700438a310
Prevent the WhereLoop.rSetup cost estimate from going negative on complex
2016-07-01 12:39:58 +00:00 · 2016-06-26 04:06:28 +00:00 · 2016-06-25 11:43:47 +00:00 · 2016-06-24 06:23:13 +00:00 · 2016-06-24 02:50:44 +00:00 · 2016-06-23 16:48:35 +00:00
63 changed files with 2947 additions and 483 deletions
@@ -417,6 +417,7 @@ TESTSRC = \
 TESTSRC += \
  $(TOP)/ext/misc/amatch.c \
  $(TOP)/ext/misc/closure.c \
+  $(TOP)/ext/misc/csv.c \
  $(TOP)/ext/misc/eval.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
@@ -538,7 +539,8 @@ TESTPROGS = \
  testfixture$(TEXE) \
  sqlite3$(TEXE) \
  sqlite3_analyzer$(TEXE) \
-  sqldiff$(TEXE)
+  sqldiff$(TEXE) \
+  dbhash$(TEXE)

 # Databases containing fuzzer test cases
 #
@@ -586,12 +588,15 @@ sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
 		$(TOP)/src/shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"

-sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
-	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
+sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
+	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)

-scrub$(TEXE):	$(TOP)/ext/misc/scrub.c sqlite3.o
+dbhash$(TEXE):	$(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h
+	$(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS)
+
+scrub$(TEXE):	$(TOP)/ext/misc/scrub.c sqlite3.lo
 	$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
-		$(TOP)/ext/misc/scrub.c sqlite3.o $(TLIBS)
+		$(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS)

 srcck1$(BEXE):	$(TOP)/tool/srcck1.c
 	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
@@ -606,8 +611,8 @@ fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)

-mptester$(TEXE):	sqlite3.c $(TOP)/mptest/mptest.c
-	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
+mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
+	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
 		$(TLIBS) -rpath "$(libdir)"

 MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
@@ -1151,8 +1156,8 @@ rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
 LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c

-wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.c
-	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS)
+wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)

 speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
@@ -1169,7 +1174,7 @@ loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
 #
 VALIDIDS=' sqlite3(changeset|changegroup|session)?_'
 checksymbols: sqlite3.lo
-	nm -g --defined-only sqlite3.o | egrep -v $(VALIDIDS); test $$? -ne 0
+	nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0
 	echo '0 errors out of 1 tests'

 # Build the amalgamation-autoconf package.  The amalamgation-tarball target builds
@@ -1250,6 +1255,7 @@ clean:
 	rm -f fuzzershell fuzzershell.exe
 	rm -f fuzzcheck fuzzcheck.exe
 	rm -f sqldiff sqldiff.exe
+	rm -f dbhash dbhash.exe
 	rm -f fts5.* fts5parse.*

 distclean:	clean
@@ -1293,6 +1293,7 @@ TESTSRC = \
 TESTEXT = \
  $(TOP)\ext\misc\amatch.c \
  $(TOP)\ext\misc\closure.c \
+  $(TOP)\ext\misc\csv.c \
  $(TOP)\ext\misc\eval.c \
  $(TOP)\ext\misc\fileio.c \
  $(TOP)\ext\misc\fuzzer.c \
@@ -1377,7 +1378,8 @@ TESTPROGS = \
  testfixture.exe \
  $(SQLITE3EXE) \
  sqlite3_analyzer.exe \
-  sqldiff.exe
+  sqldiff.exe \
+  dbhash.exe

 # Databases containing fuzzer test cases
 #
@@ -1455,6 +1457,9 @@ $(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_S
 sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

+dbhash.exe:	$(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

@@ -2104,6 +2109,6 @@ clean:
 	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
-	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
+	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
 	del /Q fts5.* fts5parse.* 2>NUL
 # <</mark>>
@@ -30,40 +30,52 @@ AC_FUNC_STRERROR_R
 AC_CONFIG_FILES([Makefile sqlite3.pc])
 AC_SUBST(BUILD_CFLAGS)

-#-----------------------------------------------------------------------
+#-------------------------------------------------------------------------
+# Two options to enable readline compatible libraries: 
+#
 #   --enable-editline
 #   --enable-readline
 #
-AC_ARG_ENABLE(editline, [AS_HELP_STRING(
-  [--enable-editline], 
-  [use BSD libedit])], 
-  [], [enable_editline=yes])
-AC_ARG_ENABLE(readline, [AS_HELP_STRING(
-  [--enable-readline], 
-  [use readline])], 
-  [], [enable_readline=yes])
-if test x"$enable_editline" != xno ; then
-  sLIBS=$LIBS
-  LIBS=""
-  AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no])
-  READLINE_LIBS=$LIBS
-  if test x"$LIBS" != "x"; then
-    AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
-    enable_readline=no
-  else
-    unset ac_cv_search_readline
-  fi
-  LIBS=$sLIBS
-fi
-if test x"$enable_readline" != xno ; then
-  sLIBS=$LIBS
-  LIBS=""
-  AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], [])
-  AC_SEARCH_LIBS(readline, readline, [], [enable_readline=no])
-  AC_CHECK_FUNCS(readline, [], [])
-  READLINE_LIBS=$LIBS
-  LIBS=$sLIBS
-fi
+# Both are enabled by default. If, after command line processing both are
+# still enabled, the script searches for editline first and automatically
+# disables readline if it is found. So, to use readline explicitly, the
+# user must pass "--disable-editline". To disable command line editing
+# support altogether, "--disable-editline --disable-readline".
+#
+# When searching for either library, check for headers before libraries 
+# as some distros supply packages that contain libraries but not header
+# files, which come as a separate development package.
+#
+AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])])
+AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])])
+
+AS_IF([ test x"$enable_editline" != xno ],[
+  AC_CHECK_HEADERS([editline/readline.h],[
+    sLIBS=$LIBS
+    LIBS=""
+    AC_SEARCH_LIBS([readline],[edit],[
+      AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
+      READLINE_LIBS=$LIBS
+      enable_readline=no
+    ])
+    AS_UNSET(ac_cv_search_readline)
+    LIBS=$sLIBS
+  ])
+])
+
+AS_IF([ test x"$enable_readline" != xno ],[
+  AC_CHECK_HEADERS([readline/readline.h],[
+    sLIBS=$LIBS
+    LIBS=""
+    AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], [])
+    AC_SEARCH_LIBS(readline,[readline edit], [
+      AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper)
+      READLINE_LIBS=$LIBS
+    ])
+    LIBS=$sLIBS
+  ])
+])
+
 AC_SUBST(READLINE_LIBS)
 #-----------------------------------------------------------------------

@@ -686,7 +686,6 @@ int sqlite3Fts5ExprPopulatePoslists(
    Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 );
 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
-void sqlite3Fts5ExprClearEof(Fts5Expr*);

 int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);

@@ -2617,17 +2617,6 @@ void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
  fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
 }

-static void fts5ExprClearEof(Fts5ExprNode *pNode){
-  int i;
-  for(i=0; i<pNode->nChild; i++){
-    fts5ExprClearEof(pNode->apChild[i]);
-  }
-  pNode->bEof = 0;
-}
-void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
-  fts5ExprClearEof(pExpr->pRoot);
-}
-
 /*
 ** This function is only called for detail=columns tables. 
 */
@@ -1186,7 +1186,6 @@ static int fts5FilterMethod(
    pCsr->ePlan = FTS5_PLAN_SOURCE;
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bDesc);
-    sqlite3Fts5ExprClearEof(pCsr->pExpr);
  }else if( pMatch ){
    const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
    if( zExpr==0 ) zExpr = "";
@@ -91,9 +91,38 @@ do_test 2.7 {
 } {1 3 2}


+#--------------------------------------------------------------------------
+# At one point there was a problem with queries such as:
+#
+#   ... MATCH 'x OR y' ORDER BY rank;
+#
+# if there were zero occurrences of token 'y' in the dataset. The
+# following tests verify that that problem has been addressed.
+#
+foreach_detail_mode $::testprefix {
+  do_execsql_test 3.0 {
+    CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%);
+    INSERT INTO y1 VALUES('test xyz');
+    INSERT INTO y1 VALUES('test test xyz test');
+    INSERT INTO y1 VALUES('test test xyz');
+  }

+  do_execsql_test 3.1 {
+    SELECT rowid FROM y1('test OR tset');
+  } {1 2 3}

+  do_execsql_test 3.2 {
+    SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1)
+  } {2 3 1}

+  do_execsql_test 3.3 {
+    SELECT rowid FROM y1('test OR tset') ORDER BY +rank
+  } {2 3 1}
+
+  do_execsql_test 3.4 {
+    SELECT rowid FROM y1('test OR tset') ORDER BY rank
+  } {2 3 1}
+}


 finish_test
@@ -0,0 +1,861 @@
+/*
+** 2016-05-28
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains the implementation of an SQLite virtual table for
+** reading CSV files.
+**
+** Usage:
+**
+**    .load ./csv
+**    CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
+**    SELECT * FROM csv;
+**
+** The columns are named "c1", "c2", "c3", ... by default.  But the
+** application can define its own CREATE TABLE statement as an additional
+** parameter.  For example:
+**
+**    CREATE VIRTUAL TABLE temp.csv2 USING csv(
+**       filename = "../http.log",
+**       schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)"
+**    );
+**
+** Instead of specifying a file, the text of the CSV can be loaded using
+** the data= parameter.
+**
+** If the columns=N parameter is supplied, then the CSV file is assumed to have
+** N columns.  If the columns parameter is omitted, the CSV file is opened
+** as soon as the virtual table is constructed and the first row of the CSV
+** is read in order to count the tables.
+**
+** Some extra debugging features (used for testing virtual tables) are available
+** if this module is compiled with -DSQLITE_TEST.
+*/
+#include <sqlite3ext.h>
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <stdio.h>
+
+/*
+** A macro to hint to the compiler that a function should not be
+** inlined.
+*/
+#if defined(__GNUC__)
+#  define CSV_NOINLINE  __attribute__((noinline))
+#elif defined(_MSC_VER) && _MSC_VER>=1310
+#  define CSV_NOINLINE  __declspec(noinline)
+#else
+#  define CSV_NOINLINE
+#endif
+
+
+/* Max size of the error message in a CsvReader */
+#define CSV_MXERR 200
+
+/* Size of the CsvReader input buffer */
+#define CSV_INBUFSZ 1024
+
+/* A context object used when read a CSV file. */
+typedef struct CsvReader CsvReader;
+struct CsvReader {
+  FILE *in;              /* Read the CSV text from this input stream */
+  char *z;               /* Accumulated text for a field */
+  int n;                 /* Number of bytes in z */
+  int nAlloc;            /* Space allocated for z[] */
+  int nLine;             /* Current line number */
+  char cTerm;            /* Character that terminated the most recent field */
+  size_t iIn;            /* Next unread character in the input buffer */
+  size_t nIn;            /* Number of characters in the input buffer */
+  char *zIn;             /* The input buffer */
+  char zErr[CSV_MXERR];  /* Error message */
+};
+
+/* Initialize a CsvReader object */
+static void csv_reader_init(CsvReader *p){
+  p->in = 0;
+  p->z = 0;
+  p->n = 0;
+  p->nAlloc = 0;
+  p->nLine = 0;
+  p->nIn = 0;
+  p->zIn = 0;
+  p->zErr[0] = 0;
+}
+
+/* Close and reset a CsvReader object */
+static void csv_reader_reset(CsvReader *p){
+  if( p->in ){
+    fclose(p->in);
+    sqlite3_free(p->zIn);
+  }
+  sqlite3_free(p->z);
+  csv_reader_init(p);
+}
+
+/* Report an error on a CsvReader */
+static void csv_errmsg(CsvReader *p, const char *zFormat, ...){
+  va_list ap;
+  va_start(ap, zFormat);
+  sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap);
+  va_end(ap);
+}
+
+/* Open the file associated with a CsvReader
+** Return the number of errors.
+*/
+static int csv_reader_open(
+  CsvReader *p,               /* The reader to open */
+  const char *zFilename,      /* Read from this filename */
+  const char *zData           /*  ... or use this data */
+){
+  if( zFilename ){
+    p->zIn = sqlite3_malloc( CSV_INBUFSZ );
+    if( p->zIn==0 ){
+      csv_errmsg(p, "out of memory");
+      return 1;
+    }
+    p->in = fopen(zFilename, "rb");
+    if( p->in==0 ){
+      csv_reader_reset(p);
+      csv_errmsg(p, "cannot open '%s' for reading", zFilename);
+      return 1;
+    }
+  }else{
+    assert( p->in==0 );
+    p->zIn = (char*)zData;
+    p->nIn = strlen(zData);
+  }
+  return 0;
+}
+
+/* The input buffer has overflowed.  Refill the input buffer, then
+** return the next character
+*/
+static CSV_NOINLINE int csv_getc_refill(CsvReader *p){
+  size_t got;
+
+  assert( p->iIn>=p->nIn );  /* Only called on an empty input buffer */
+  assert( p->in!=0 );        /* Only called if reading froma file */
+
+  got = fread(p->zIn, 1, CSV_INBUFSZ, p->in);
+  if( got==0 ) return EOF;
+  p->nIn = got;
+  p->iIn = 1;
+  return p->zIn[0];
+}
+
+/* Return the next character of input.  Return EOF at end of input. */
+static int csv_getc(CsvReader *p){
+  if( p->iIn >= p->nIn ){
+    if( p->in!=0 ) return csv_getc_refill(p);
+    return EOF;
+  }
+  return p->zIn[p->iIn++];
+}
+
+/* Increase the size of p->z and append character c to the end. 
+** Return 0 on success and non-zero if there is an OOM error */
+static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){
+  char *zNew;
+  int nNew = p->nAlloc*2 + 100;
+  zNew = sqlite3_realloc64(p->z, nNew);
+  if( zNew ){
+    p->z = zNew;
+    p->nAlloc = nNew;
+    p->z[p->n++] = c;
+    return 0;
+  }else{
+    csv_errmsg(p, "out of memory");
+    return 1;
+  }
+}
+
+/* Append a single character to the CsvReader.z[] array.
+** Return 0 on success and non-zero if there is an OOM error */
+static int csv_append(CsvReader *p, char c){
+  if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c);
+  p->z[p->n++] = c;
+  return 0;
+}
+
+/* Read a single field of CSV text.  Compatible with rfc4180 and extended
+** with the option of having a separator other than ",".
+**
+**   +  Input comes from p->in.
+**   +  Store results in p->z of length p->n.  Space to hold p->z comes
+**      from sqlite3_malloc64().
+**   +  Keep track of the line number in p->nLine.
+**   +  Store the character that terminates the field in p->cTerm.  Store
+**      EOF on end-of-file.
+**
+** Return "" at EOF.  Return 0 on an OOM error.
+*/
+static char *csv_read_one_field(CsvReader *p){
+  int c;
+  p->n = 0;
+  c = csv_getc(p);
+  if( c==EOF ){
+    p->cTerm = EOF;
+    return "";
+  }
+  if( c=='"' ){
+    int pc, ppc;
+    int startLine = p->nLine;
+    pc = ppc = 0;
+    while( 1 ){
+      c = csv_getc(p);
+      if( c<='"' || pc=='"' ){
+        if( c=='\n' ) p->nLine++;
+        if( c=='"' ){
+          if( pc=='"' ){
+            pc = 0;
+            continue;
+          }
+        }
+        if( (c==',' && pc=='"')
+         || (c=='\n' && pc=='"')
+         || (c=='\n' && pc=='\r' && ppc=='"')
+         || (c==EOF && pc=='"')
+        ){
+          do{ p->n--; }while( p->z[p->n]!='"' );
+          p->cTerm = c;
+          break;
+        }
+        if( pc=='"' && c!='\r' ){
+          csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"');
+          break;
+        }
+        if( c==EOF ){
+          csv_errmsg(p, "line %d: unterminated %c-quoted field\n",
+                     startLine, '"');
+          p->cTerm = c;
+          break;
+        }
+      }
+      if( csv_append(p, (char)c) ) return 0;
+      ppc = pc;
+      pc = c;
+    }
+  }else{
+    while( c>',' || (c!=EOF && c!=',' && c!='\n') ){
+      if( csv_append(p, (char)c) ) return 0;
+      c = csv_getc(p);
+    }
+    if( c=='\n' ){
+      p->nLine++;
+      if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--;
+    }
+    p->cTerm = c;
+  }
+  if( p->z ) p->z[p->n] = 0;
+  return p->z;
+}
+
+
+/* Forward references to the various virtual table methods implemented
+** in this file. */
+static int csvtabCreate(sqlite3*, void*, int, const char*const*, 
+                           sqlite3_vtab**,char**);
+static int csvtabConnect(sqlite3*, void*, int, const char*const*, 
+                           sqlite3_vtab**,char**);
+static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*);
+static int csvtabDisconnect(sqlite3_vtab*);
+static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**);
+static int csvtabClose(sqlite3_vtab_cursor*);
+static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr,
+                          int argc, sqlite3_value **argv);
+static int csvtabNext(sqlite3_vtab_cursor*);
+static int csvtabEof(sqlite3_vtab_cursor*);
+static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int);
+static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*);
+
+/* An instance of the CSV virtual table */
+typedef struct CsvTable {
+  sqlite3_vtab base;              /* Base class.  Must be first */
+  char *zFilename;                /* Name of the CSV file */
+  char *zData;                    /* Raw CSV data in lieu of zFilename */
+  long iStart;                    /* Offset to start of data in zFilename */
+  int nCol;                       /* Number of columns in the CSV file */
+  unsigned int tstFlags;          /* Bit values used for testing */
+} CsvTable;
+
+/* Allowed values for tstFlags */
+#define CSVTEST_FIDX  0x0001      /* Pretend that constrained searchs cost less*/
+
+/* A cursor for the CSV virtual table */
+typedef struct CsvCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  CsvReader rdr;                  /* The CsvReader object */
+  char **azVal;                   /* Value of the current row */
+  int *aLen;                      /* Length of each entry */
+  sqlite3_int64 iRowid;           /* The current rowid.  Negative for EOF */
+} CsvCursor;
+
+/* Transfer error message text from a reader into a CsvTable */
+static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){
+  sqlite3_free(pTab->base.zErrMsg);
+  pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr);
+}
+
+/*
+** This method is the destructor fo a CsvTable object.
+*/
+static int csvtabDisconnect(sqlite3_vtab *pVtab){
+  CsvTable *p = (CsvTable*)pVtab;
+  sqlite3_free(p->zFilename);
+  sqlite3_free(p->zData);
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/* Skip leading whitespace.  Return a pointer to the first non-whitespace
+** character, or to the zero terminator if the string has only whitespace */
+static const char *csv_skip_whitespace(const char *z){
+  while( isspace((unsigned char)z[0]) ) z++;
+  return z;
+}
+
+/* Remove trailing whitespace from the end of string z[] */
+static void csv_trim_whitespace(char *z){
+  size_t n = strlen(z);
+  while( n>0 && isspace((unsigned char)z[n]) ) n--;
+  z[n] = 0;
+}
+
+/* Dequote the string */
+static void csv_dequote(char *z){
+  int i, j;
+  char cQuote = z[0];
+  size_t n;
+
+  if( cQuote!='\'' && cQuote!='"' ) return;
+  n = strlen(z);
+  if( n<2 || z[n-1]!=z[0] ) return;
+  for(i=1, j=0; i<n-1; i++){
+    if( z[i]==cQuote && z[i+1]==cQuote ) i++;
+    z[j++] = z[i];
+  }
+  z[j] = 0;
+}
+
+/* Check to see if the string is of the form:  "TAG = VALUE" with optional
+** whitespace before and around tokens.  If it is, return a pointer to the
+** first character of VALUE.  If it is not, return NULL.
+*/
+static const char *csv_parameter(const char *zTag, int nTag, const char *z){
+  z = csv_skip_whitespace(z);
+  if( strncmp(zTag, z, nTag)!=0 ) return 0;
+  z = csv_skip_whitespace(z+nTag);
+  if( z[0]!='=' ) return 0;
+  return csv_skip_whitespace(z+1);
+}
+
+/* Decode a parameter that requires a dequoted string.
+**
+** Return 1 if the parameter is seen, or 0 if not.  1 is returned
+** even if there is an error.  If an error occurs, then an error message
+** is left in p->zErr.  If there are no errors, p->zErr[0]==0.
+*/
+static int csv_string_parameter(
+  CsvReader *p,            /* Leave the error message here, if there is one */
+  const char *zParam,      /* Parameter we are checking for */
+  const char *zArg,        /* Raw text of the virtual table argment */
+  char **pzVal             /* Write the dequoted string value here */
+){
+  const char *zValue;
+  zValue = csv_parameter(zParam,strlen(zParam),zArg);
+  if( zValue==0 ) return 0;
+  p->zErr[0] = 0;
+  if( *pzVal ){
+    csv_errmsg(p, "more than one '%s' parameter", zParam);
+    return 1;
+  }
+  *pzVal = sqlite3_mprintf("%s", zValue);
+  if( *pzVal==0 ){
+    csv_errmsg(p, "out of memory");
+    return 1;
+  }
+  csv_trim_whitespace(*pzVal);
+  csv_dequote(*pzVal);
+  return 1;
+}
+
+
+/* Return 0 if the argument is false and 1 if it is true.  Return -1 if
+** we cannot really tell.
+*/
+static int csv_boolean(const char *z){
+  if( sqlite3_stricmp("yes",z)==0
+   || sqlite3_stricmp("on",z)==0
+   || sqlite3_stricmp("true",z)==0
+   || (z[0]=='1' && z[0]==0)
+  ){
+    return 1;
+  }
+  if( sqlite3_stricmp("no",z)==0
+   || sqlite3_stricmp("off",z)==0
+   || sqlite3_stricmp("false",z)==0
+   || (z[0]=='0' && z[1]==0)
+  ){
+    return 0;
+  }
+  return -1;
+}
+
+
+/*
+** Parameters:
+**    filename=FILENAME          Name of file containing CSV content
+**    data=TEXT                  Direct CSV content.
+**    schema=SCHEMA              Alternative CSV schema.
+**    header=YES|NO              First row of CSV defines the names of
+**                               columns if "yes".  Default "no".
+**    columns=N                  Assume the CSV file contains N columns.
+**
+** Only available if compiled with SQLITE_TEST:
+**    
+**    testflags=N                Bitmask of test flags.  Optional
+**
+** If schema= is omitted, then the columns are named "c0", "c1", "c2",
+** and so forth.  If columns=N is omitted, then the file is opened and
+** the number of columns in the first row is counted to determine the
+** column count.  If header=YES, then the first row is skipped.
+*/
+static int csvtabConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  CsvTable *pNew = 0;        /* The CsvTable object to construct */
+  int bHeader = -1;          /* header= flags.  -1 means not seen yet */
+  int rc = SQLITE_OK;        /* Result code from this routine */
+  int i, j;                  /* Loop counters */
+#ifdef SQLITE_TEST
+  int tstFlags = 0;          /* Value for testflags=N parameter */
+#endif
+  int nCol = -99;            /* Value of the columns= parameter */
+  CsvReader sRdr;            /* A CSV file reader used to store an error
+                             ** message and/or to count the number of columns */
+  static const char *azParam[] = {
+     "filename", "data", "schema", 
+  };
+  char *azPValue[3];         /* Parameter values */
+# define CSV_FILENAME (azPValue[0])
+# define CSV_DATA     (azPValue[1])
+# define CSV_SCHEMA   (azPValue[2])
+
+
+  assert( sizeof(azPValue)==sizeof(azParam) );
+  memset(&sRdr, 0, sizeof(sRdr));
+  memset(azPValue, 0, sizeof(azPValue));
+  for(i=3; i<argc; i++){
+    const char *z = argv[i];
+    const char *zValue;
+    for(j=0; j<sizeof(azParam)/sizeof(azParam[0]); j++){
+      if( csv_string_parameter(&sRdr, azParam[j], z, &azPValue[j]) ) break;
+    }
+    if( j<sizeof(azParam)/sizeof(azParam[0]) ){
+      if( sRdr.zErr[0] ) goto csvtab_connect_error;
+    }else
+    if( (zValue = csv_parameter("header",6,z))!=0 ){
+      int x;
+      if( bHeader>=0 ){
+        csv_errmsg(&sRdr, "more than one 'header' parameter");
+        goto csvtab_connect_error;
+      }
+      x = csv_boolean(zValue);
+      if( x==1 ){
+        bHeader = 1;
+      }else if( x==0 ){
+        bHeader = 0;
+      }else{
+        csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue);
+        goto csvtab_connect_error;
+      }
+    }else
+#ifdef SQLITE_TEST
+    if( (zValue = csv_parameter("testflags",9,z))!=0 ){
+      tstFlags = (unsigned int)atoi(zValue);
+    }else
+#endif
+    if( (zValue = csv_parameter("columns",7,z))!=0 ){
+      if( nCol>0 ){
+        csv_errmsg(&sRdr, "more than one 'columns' parameter");
+        goto csvtab_connect_error;
+      }
+      nCol = atoi(zValue);
+      if( nCol<=0 ){
+        csv_errmsg(&sRdr, "must have at least one column");
+        goto csvtab_connect_error;
+      }
+    }else
+    {
+      csv_errmsg(&sRdr, "unrecognized parameter '%s'", z);
+      goto csvtab_connect_error;
+    }
+  }
+  if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
+    csv_errmsg(&sRdr, "must either filename= or data= but not both");
+    goto csvtab_connect_error;
+  }
+  if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){
+    goto csvtab_connect_error;
+  }
+  pNew = sqlite3_malloc( sizeof(*pNew) );
+  *ppVtab = (sqlite3_vtab*)pNew;
+  if( pNew==0 ) goto csvtab_connect_oom;
+  memset(pNew, 0, sizeof(*pNew));
+  if( nCol>0 ){
+    pNew->nCol = nCol;
+  }else{
+    do{
+      const char *z = csv_read_one_field(&sRdr);
+      if( z==0 ) goto csvtab_connect_oom;
+      pNew->nCol++;
+    }while( sRdr.cTerm==',' );
+  }
+  pNew->zFilename = CSV_FILENAME;  CSV_FILENAME = 0;
+  pNew->zData = CSV_DATA;          CSV_DATA = 0;
+#ifdef SQLITE_TEST
+  pNew->tstFlags = tstFlags;
+#endif
+  pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0;
+  csv_reader_reset(&sRdr);
+  if( CSV_SCHEMA==0 ){
+    char *zSep = "";
+    CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x(");
+    if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
+    for(i=0; i<pNew->nCol; i++){
+      CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i);
+      zSep = ",";
+    }
+    CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA);
+  }
+  rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
+  if( rc ) goto csvtab_connect_error;
+  for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
+    sqlite3_free(azPValue[i]);
+  }
+  return SQLITE_OK;
+
+csvtab_connect_oom:
+  rc = SQLITE_NOMEM;
+  csv_errmsg(&sRdr, "out of memory");
+
+csvtab_connect_error:
+  if( pNew ) csvtabDisconnect(&pNew->base);
+  for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
+    sqlite3_free(azPValue[i]);
+  }
+  if( sRdr.zErr[0] ){
+    sqlite3_free(*pzErr);
+    *pzErr = sqlite3_mprintf("%s", sRdr.zErr);
+  }
+  csv_reader_reset(&sRdr);
+  if( rc==SQLITE_OK ) rc = SQLITE_ERROR;
+  return rc;
+}
+
+/*
+** Reset the current row content held by a CsvCursor.
+*/
+static void csvtabCursorRowReset(CsvCursor *pCur){
+  CsvTable *pTab = (CsvTable*)pCur->base.pVtab;
+  int i;
+  for(i=0; i<pTab->nCol; i++){
+    sqlite3_free(pCur->azVal[i]);
+    pCur->azVal[i] = 0;
+    pCur->aLen[i] = 0;
+  }
+}
+
+/*
+** The xConnect and xCreate methods do the same thing, but they must be
+** different so that the virtual table is not an eponymous virtual table.
+*/
+static int csvtabCreate(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+ return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr);
+}
+
+/*
+** Destructor for a CsvCursor.
+*/
+static int csvtabClose(sqlite3_vtab_cursor *cur){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  csvtabCursorRowReset(pCur);
+  csv_reader_reset(&pCur->rdr);
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new CsvTable cursor object.
+*/
+static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  CsvTable *pTab = (CsvTable*)p;
+  CsvCursor *pCur;
+  size_t nByte;
+  nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol;
+  pCur = sqlite3_malloc( nByte );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, nByte);
+  pCur->azVal = (char**)&pCur[1];
+  pCur->aLen = (int*)&pCur->azVal[pTab->nCol];
+  *ppCursor = &pCur->base;
+  if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){
+    csv_xfer_error(pTab, &pCur->rdr);
+    return SQLITE_ERROR;
+  }
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a CsvCursor to its next row of input.
+** Set the EOF marker if we reach the end of input.
+*/
+static int csvtabNext(sqlite3_vtab_cursor *cur){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  CsvTable *pTab = (CsvTable*)cur->pVtab;
+  int i = 0;
+  char *z;
+  do{
+    z = csv_read_one_field(&pCur->rdr);
+    if( z==0 ){
+      csv_xfer_error(pTab, &pCur->rdr);
+      break;
+    }
+    if( i<pTab->nCol ){
+      if( pCur->aLen[i] < pCur->rdr.n+1 ){
+        char *zNew = sqlite3_realloc(pCur->azVal[i], pCur->rdr.n+1);
+        if( zNew==0 ){
+          csv_errmsg(&pCur->rdr, "out of memory");
+          csv_xfer_error(pTab, &pCur->rdr);
+          break;
+        }
+        pCur->azVal[i] = zNew;
+        pCur->aLen[i] = pCur->rdr.n+1;
+      }
+      memcpy(pCur->azVal[i], z, pCur->rdr.n+1);
+      i++;
+    }
+  }while( pCur->rdr.cTerm==',' );
+  while( i<pTab->nCol ){
+    sqlite3_free(pCur->azVal[i]);
+    pCur->azVal[i] = 0;
+    pCur->aLen[i] = 0;
+    i++;
+  }
+  if( z==0 || pCur->rdr.cTerm==EOF ){
+    pCur->iRowid = -1;
+  }else{
+    pCur->iRowid++;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the CsvCursor
+** is currently pointing.
+*/
+static int csvtabColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  CsvTable *pTab = (CsvTable*)cur->pVtab;
+  if( i>=0 && i<pTab->nCol && pCur->azVal[i]!=0 ){
+    sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_STATIC);
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.
+*/
+static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int csvtabEof(sqlite3_vtab_cursor *cur){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  return pCur->iRowid<0;
+}
+
+/*
+** Only a full table scan is supported.  So xFilter simply rewinds to
+** the beginning.
+*/
+static int csvtabFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  CsvCursor *pCur = (CsvCursor*)pVtabCursor;
+  CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab;
+  pCur->iRowid = 0;
+  if( pCur->rdr.in==0 ){
+    assert( pCur->rdr.zIn==pTab->zData );
+    assert( pTab->iStart<=pCur->rdr.nIn );
+    pCur->rdr.iIn = pTab->iStart;
+  }else{
+    fseek(pCur->rdr.in, pTab->iStart, SEEK_SET);
+    pCur->rdr.iIn = 0;
+    pCur->rdr.nIn = 0;
+  }
+  return csvtabNext(pVtabCursor);
+}
+
+/*
+** Only a forward full table scan is supported.  xBestIndex is mostly
+** a no-op.  If CSVTEST_FIDX is set, then the presence of equality
+** constraints lowers the estimated cost, which is fiction, but is useful
+** for testing certain kinds of virtual table behavior.
+*/
+static int csvtabBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = 1000000;
+#ifdef SQLITE_TEST
+  if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){
+    /* The usual (and sensible) case is to always do a full table scan.
+    ** The code in this branch only runs when testflags=1.  This code
+    ** generates an artifical and unrealistic plan which is useful
+    ** for testing virtual table logic but is not helpful to real applications.
+    **
+    ** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual
+    ** table (even though it is not) and the cost of running the virtual table
+    ** is reduced from 1 million to just 10.  The constraints are *not* marked
+    ** as omittable, however, so the query planner should still generate a
+    ** plan that gives a correct answer, even if they plan is not optimal.
+    */
+    int i;
+    int nConst = 0;
+    for(i=0; i<pIdxInfo->nConstraint; i++){
+      unsigned char op;
+      if( pIdxInfo->aConstraint[i].usable==0 ) continue;
+      op = pIdxInfo->aConstraint[i].op;
+      if( op==SQLITE_INDEX_CONSTRAINT_EQ 
+       || op==SQLITE_INDEX_CONSTRAINT_LIKE
+       || op==SQLITE_INDEX_CONSTRAINT_GLOB
+      ){
+        pIdxInfo->estimatedCost = 10;
+        pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1;
+        nConst++;
+      }
+    }
+  }
+#endif
+  return SQLITE_OK;
+}
+
+
+static sqlite3_module CsvModule = {
+  0,                       /* iVersion */
+  csvtabCreate,            /* xCreate */
+  csvtabConnect,           /* xConnect */
+  csvtabBestIndex,         /* xBestIndex */
+  csvtabDisconnect,        /* xDisconnect */
+  csvtabDisconnect,        /* xDestroy */
+  csvtabOpen,              /* xOpen - open a cursor */
+  csvtabClose,             /* xClose - close a cursor */
+  csvtabFilter,            /* xFilter - configure scan constraints */
+  csvtabNext,              /* xNext - advance a cursor */
+  csvtabEof,               /* xEof - check for end of scan */
+  csvtabColumn,            /* xColumn - read data */
+  csvtabRowid,             /* xRowid - read data */
+  0,                       /* xUpdate */
+  0,                       /* xBegin */
+  0,                       /* xSync */
+  0,                       /* xCommit */
+  0,                       /* xRollback */
+  0,                       /* xFindMethod */
+  0,                       /* xRename */
+};
+
+#ifdef SQLITE_TEST
+/*
+** For virtual table testing, make a version of the CSV virtual table
+** available that has an xUpdate function.  But the xUpdate always returns
+** SQLITE_READONLY since the CSV file is not really writable.
+*/
+static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){
+  return SQLITE_READONLY;
+}
+static sqlite3_module CsvModuleFauxWrite = {
+  0,                       /* iVersion */
+  csvtabCreate,            /* xCreate */
+  csvtabConnect,           /* xConnect */
+  csvtabBestIndex,         /* xBestIndex */
+  csvtabDisconnect,        /* xDisconnect */
+  csvtabDisconnect,        /* xDestroy */
+  csvtabOpen,              /* xOpen - open a cursor */
+  csvtabClose,             /* xClose - close a cursor */
+  csvtabFilter,            /* xFilter - configure scan constraints */
+  csvtabNext,              /* xNext - advance a cursor */
+  csvtabEof,               /* xEof - check for end of scan */
+  csvtabColumn,            /* xColumn - read data */
+  csvtabRowid,             /* xRowid - read data */
+  csvtabUpdate,            /* xUpdate */
+  0,                       /* xBegin */
+  0,                       /* xSync */
+  0,                       /* xCommit */
+  0,                       /* xRollback */
+  0,                       /* xFindMethod */
+  0,                       /* xRename */
+};
+#endif /* SQLITE_TEST */
+
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+/* 
+** This routine is called when the extension is loaded.  The new
+** CSV virtual table module is registered with the calling database
+** connection.
+*/
+int sqlite3_csv_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc;
+  SQLITE_EXTENSION_INIT2(pApi);
+  rc = sqlite3_create_module(db, "csv", &CsvModule, 0);
+#ifdef SQLITE_TEST
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0);
+  }
+#endif
+  return rc;
+}
@@ -402,7 +402,7 @@ static int vstatShmLock(sqlite3_file *pFile, int offset, int n, int flags){
 /* Memory barrier operation on shared memory */
 static void vstatShmBarrier(sqlite3_file *pFile){
  VStatFile *p = (VStatFile *)pFile;
-  return p->pReal->pMethods->xShmBarrier(p->pReal);
+  p->pReal->pMethods->xShmBarrier(p->pReal);
 }

 /* Unmap a shared memory segment */
@@ -804,16 +804,13 @@ int sqlite3_vfsstat_init(
  const sqlite3_api_routines *pApi
 ){
  int rc = SQLITE_OK;
-  VStatVfs *pNew;
  SQLITE_EXTENSION_INIT2(pApi);
-  pNew = sqlite3_malloc(sizeof(vstat_vfs));
-  if( pNew==0 ) return SQLITE_NOMEM;
-  memcpy(&pNew->base, &vstat_vfs, sizeof(vstat_vfs));
-  pNew->pVfs = sqlite3_vfs_find(0);
-  pNew->base.szOsFile = sizeof(VStatFile) + pNew->pVfs->szOsFile;
-  rc = sqlite3_vfs_register(&pNew->base, 1);
+  vstat_vfs.pVfs = sqlite3_vfs_find(0);
+  vstat_vfs.base.szOsFile = sizeof(VStatFile) + vstat_vfs.pVfs->szOsFile;
+  rc = sqlite3_vfs_register(&vstat_vfs.base, 1);
  if( rc==SQLITE_OK ){
    rc = sqlite3_auto_extension((void(*)(void))vstatRegister);
  }
+  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
  return rc;
 }
@@ -10,10 +10,7 @@
 #***********************************************************************
 #

-if {![info exists testdir]} {
-  set testdir [file join [file dirname [info script]] .. .. test]
-}
-source $testdir/tester.tcl
+source [file join [file dirname [info script]] rbu_common.tcl]
 set ::testprefix rbu1

 db close
@@ -96,26 +93,6 @@ proc create_rbu5 {filename} {
  return $filename
 }

-# Run the RBU in file $rbu on target database $target until completion.
-#
-proc run_rbu {target rbu} {
-  sqlite3rbu rbu $target $rbu
-  while 1 {
-    set rc [rbu step]
-    if {$rc!="SQLITE_OK"} break
-  }
-  rbu close
-}
-
-proc step_rbu {target rbu} {
-  while 1 {
-    sqlite3rbu rbu $target $rbu
-    set rc [rbu step]
-    rbu close
-    if {$rc != "SQLITE_OK"} break
-  }
-  set rc
-}

 # Same as [step_rbu], except using a URI to open the target db.
 #
@@ -641,10 +618,25 @@ foreach {tn3 create_vfs destroy_vfs} {
  # correctly.
  reset_db
  forcedelete rbu.db
-  do_test $tn3.8 {
+  do_test $tn3.8.1 {
    list [catch { run_rbu test.db rbu.db } msg] $msg
  } {0 SQLITE_DONE}
-  
+
+  # Test that an RBU database containing only empty data_xxx tables is
+  # also handled correctly.
+  reset_db
+  forcedelete rbu.db
+  do_execsql_test $tn3.8.2.1 {
+    CREATE TABLE t1(a PRIMARY KEY, b);
+    INSERT INTO t1 VALUES(1, 2);
+    ATTACH 'rbu.db' AS rbu;
+    CREATE TABLE data_t1(a, b, rbu_control);
+    DETACH rbu;
+  }
+  do_test $tn3.8.2.1 {
+    list [catch { run_rbu test.db rbu.db } msg] $msg
+  } {0 SQLITE_DONE}
+
  # Test that RBU can update indexes containing NULL values.
  #
  reset_db
@@ -12,35 +12,10 @@
 # Test some properties of the pager_rbu_mode and rbu_mode pragmas.
 #

-if {![info exists testdir]} {
-  set testdir [file join [file dirname [info script]] .. .. test]
-}
-source $testdir/tester.tcl
+source [file join [file dirname [info script]] rbu_common.tcl]
 set ::testprefix rbu5


-# Run the RBU in file $rbu on target database $target until completion.
-#
-proc run_rbu {target rbu} {
-  sqlite3rbu rbu $target $rbu
-  while { [rbu step]=="SQLITE_OK" } {}
-  rbu close
-}
-
-
-# Run the RBU in file $rbu on target database $target one step at a
-# time until completion.
-#
-proc step_rbu {target rbu} {
-  while 1 {
-    sqlite3rbu rbu $target $rbu
-    set rc [rbu step]
-    rbu close
-    if {$rc != "SQLITE_OK"} break
-  }
-  set rc
-}
-
 # Return a list of the primary key columns for table $tbl in the database
 # opened by database handle $db.
 #
@@ -15,12 +15,43 @@ if {![info exists testdir]} {
 }
 source $testdir/tester.tcl

+proc check_prestep_state {target state} {
+  set oal_exists [file exists $target-oal]
+  set wal_exists [file exists $target-wal]
+  set progress [rbu progress]
+
+  if {($progress==0 && $state!="oal" && $state!="done")
+   || ($oal_exists && $wal_exists)
+   || ($progress>0 && $state=="oal" && (!$oal_exists || $wal_exists))
+   || ($state=="move" && (!$oal_exists || $wal_exists))
+   || ($state=="checkpoint" && ($oal_exists || !$wal_exists))
+   || ($state=="done" && ($oal_exists && $progress!=0))
+  } {
+    error "B: state=$state progress=$progress oal=$oal_exists wal=$wal_exists"
+  }
+}
+
+proc check_poststep_state {rc target state} {
+  if {$rc=="SQLITE_OK" || $rc=="SQLITE_DONE"} {
+    set oal_exists [file exists $target-oal]
+    set wal_exists [file exists $target-wal]
+    if {$state=="move" && ($oal_exists || !$wal_exists)} {
+      error "A: state=$state progress=$progress oal=$oal_exists wal=$wal_exists"
+    }
+  }
+}
+
 # Run the RBU in file $rbu on target database $target until completion.
 #
 proc run_rbu {target rbu} {
  sqlite3rbu rbu $target $rbu
  while 1 {
+    set state [rbu state]
+
+    check_prestep_state $target $state
    set rc [rbu step]
+    check_poststep_state $rc $target $state
+
    if {$rc!="SQLITE_OK"} break
  }
  rbu close
@@ -29,10 +60,33 @@ proc run_rbu {target rbu} {
 proc step_rbu {target rbu} {
  while 1 {
    sqlite3rbu rbu $target $rbu
+    set state [rbu state]
+    check_prestep_state $target $state
    set rc [rbu step]
+    check_poststep_state $rc $target $state
    rbu close
    if {$rc != "SQLITE_OK"} break
  }
  set rc
 }

+proc do_rbu_vacuum_test {tn step} {
+  uplevel [list do_test $tn.1 {
+    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
+    while 1 {
+      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
+      set state [rbu state]
+      check_prestep_state test.db $state
+      set rc [rbu step]
+      check_poststep_state $rc test.db $state
+      if {$rc!="SQLITE_OK"} break
+      if {$step==1} { rbu close }
+    }
+    rbu close
+  } {SQLITE_DONE}]
+
+  uplevel [list do_execsql_test $tn.2 {
+    PRAGMA integrity_check
+  } ok]
+}
+
@@ -25,6 +25,10 @@ proc get_rbudiff_sql {db1 db2} {
  exec $::PROG --rbu $db1 $db2
 }

+proc get_vtab_rbudiff_sql {db1 db2} {
+  exec $::PROG --vtab --rbu $db1 $db2
+}
+
 proc step_rbu {target rbu} {
  while 1 {
    sqlite3rbu rbu $target $rbu
@@ -44,6 +48,11 @@ proc apply_rbudiff {sql target} {
  step_rbu $target rbu.db
 }

+proc sqlesc {id} {
+  set ret "'[string map {' ''} $id]'"
+  set ret
+}
+
 # The only argument is the output of an [sqldiff -rbu] run. This command
 # tests that the contents of the rbu_count table is correct. An exception
 # is thrown if it is not.
@@ -54,7 +63,7 @@ proc test_rbucount {sql} {
  tmpdb eval {
    SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table'
  } {
-    set a [tmpdb eval "SELECT count(*) FROM $name"]
+    set a [tmpdb eval "SELECT count(*) FROM [sqlesc $name]"]
    set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}]
    if {$a != $b} { 
      tmpdb close
@@ -71,9 +80,11 @@ proc rbudiff_cksum {db1} {
  sqlite3 dbtmp $db1
  foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] {
    set cols [list]
-    dbtmp eval "PRAGMA table_info = $tbl" { lappend cols "quote( $name )" }
+    dbtmp eval "PRAGMA table_info = [sqlesc $tbl]" { 
+      lappend cols "quote( $name )" 
+    }
    append txt [dbtmp eval \
-      "SELECT [join $cols {||'.'||}] FROM $tbl ORDER BY 1"
+      "SELECT [join $cols {||'.'||}] FROM [sqlesc $tbl] ORDER BY 1"
    ]
  }
  dbtmp close
@@ -159,5 +170,57 @@ foreach {tn init mod} {
  do_test 1.$tn.5 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
 }

+#-------------------------------------------------------------------------
+# Test that if the --vtab switch is present, [sqldiff] handles virtual
+# table types fts[345] and rtree correctly.
+#
+ifcapable fts3&&fts5&&rtree {
+
+foreach {tn init mod} {
+  1 {
+    CREATE VIRTUAL TABLE t1 USING fts5(c);
+    INSERT INTO t1 VALUES('a b c');
+    INSERT INTO t1 VALUES('a b c');
+  } {
+    DELETE FROM t1 WHERE rowid = 1;
+    INSERT INTO t1 VALUES('a b c');
+  }
+
+  2 {
+    CREATE VIRTUAL TABLE "x y" USING 'rtree'(id, x1, x2);
+    INSERT INTO "x y" VALUES(1, 2, 3);
+    INSERT INTO "x y" VALUES(2, 4, 6);
+  } {
+    DELETE FROM "x y" WHERE rowid = 1;
+    INSERT INTO "x y" VALUES(3, 6, 9);
+  }
+
+  3 {
+    CREATE VIRTUAL TABLE 'x''y' USING fts3;
+    INSERT INTO 'x''y' VALUES('one two three');
+    INSERT INTO 'x''y' VALUES('four five six');
+  } {
+    DELETE FROM 'x''y' WHERE rowid = 1;
+    INSERT INTO 'x''y' VALUES('one two three');
+  }
+
+} {
+
+  forcedelete test.db test.db2
+  sqlite3 db test.db
+  db eval "$init"
+  sqlite3 db test.db2
+  db eval "$init ; $mod"
+  db close
+
+  do_test 2.$tn.1 {
+    set sql [get_vtab_rbudiff_sql test.db test.db2]
+    apply_rbudiff $sql test.db
+  } {SQLITE_DONE}
+  do_test 2.$tn.2 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
+}
+
+}
+
 finish_test

@@ -17,23 +17,6 @@
 source [file join [file dirname [info script]] rbu_common.tcl]
 set ::testprefix rbuvacuum

-proc do_rbu_vacuum_test {tn step} {
-  uplevel [list do_test $tn.1 {
-    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
-    while 1 {
-      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
-      set rc [rbu step]
-      if {$rc!="SQLITE_OK"} break
-      if {$step==1} { rbu close }
-    }
-    rbu close
-  } {SQLITE_DONE}]
-
-  uplevel [list do_execsql_test $tn.2 {
-    PRAGMA integrity_check
-  } ok]
-}
-
 foreach step {0 1} {

  set ::testprefix rbuvacuum-step=$step
@@ -404,7 +387,6 @@ do_test 3.5 {
  list [catch { rbu close } msg] $msg
 } {0 SQLITE_DONE}

-
 catch { db close }
 finish_test

@@ -0,0 +1,162 @@
+# 2016 June 1
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the RBU module. More specifically, it
+# contains tests to ensure that the sqlite3rbu_vacuum() API works as
+# expected.
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+
+foreach step {0 1} {
+  set ::testprefix rbuvacuum2-$step
+  
+  #-------------------------------------------------------------------------
+  # Test that a database that contains fts3 tables can be vacuumed.
+  #
+  ifcapable fts3 {
+    reset_db
+    do_execsql_test 1.1 {
+      CREATE VIRTUAL TABLE t1 USING fts3(z, y);
+      INSERT INTO t1 VALUES('fix this issue', 'at some point');
+    }
+  
+    do_rbu_vacuum_test 1.2 $step
+  
+    do_execsql_test 1.3 {
+      SELECT * FROM t1;
+    } {{fix this issue} {at some point}}
+  
+    do_execsql_test 1.4 {
+      SELECT rowid FROM t1 WHERE t1 MATCH 'fix';
+    } {1}
+
+    do_execsql_test 1.5 {
+      INSERT INTO t1 VALUES('a b c', 'd e f');
+      INSERT INTO t1 VALUES('l h i', 'd e f');
+      DELETE FROM t1 WHERE docid = 2;
+      INSERT INTO t1 VALUES('a b c', 'x y z');
+    }
+
+    do_rbu_vacuum_test 1.6 $step
+    do_execsql_test 1.7 {
+      INSERT INTO t1(t1) VALUES('integrity-check');
+      SELECT * FROM t1;
+    } {
+      {fix this issue} {at some point}
+      {l h i} {d e f}
+      {a b c} {x y z}
+    }
+  }
+  
+  #-------------------------------------------------------------------------
+  # Test that a database that contains fts5 tables can be vacuumed.
+  #
+  ifcapable fts5 {
+    reset_db
+    do_execsql_test 2.1 {
+      CREATE VIRTUAL TABLE t1 USING fts5(z, y);
+      INSERT INTO t1 VALUES('fix this issue', 'at some point');
+    }
+  
+    do_rbu_vacuum_test 2.2 $step
+  
+    do_execsql_test 2.3 {
+      SELECT * FROM t1;
+    } {{fix this issue} {at some point}}
+  
+    do_execsql_test 2.4 {
+      SELECT rowid FROM t1 ('fix');
+    } {1}
+
+    do_execsql_test 2.5 {
+      INSERT INTO t1 VALUES('a b c', 'd e f');
+      INSERT INTO t1 VALUES('l h i', 'd e f');
+      DELETE FROM t1 WHERE rowid = 2;
+      INSERT INTO t1 VALUES('a b c', 'x y z');
+    }
+
+    do_rbu_vacuum_test 2.6 $step
+    do_execsql_test 2.7 {
+      INSERT INTO t1(t1) VALUES('integrity-check');
+      SELECT * FROM t1;
+    } {
+      {fix this issue} {at some point}
+      {l h i} {d e f}
+      {a b c} {x y z}
+    }
+  }
+
+  #-------------------------------------------------------------------------
+  # Test that a database that contains an rtree table can be vacuumed.
+  #
+  ifcapable rtree {
+    reset_db
+    do_execsql_test 3.1 {
+      CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2);
+      INSERT INTO rt VALUES(1, 45, 55);
+      INSERT INTO rt VALUES(2, 50, 60);
+      INSERT INTO rt VALUES(3, 55, 65);
+    }
+  
+    do_rbu_vacuum_test 3.2 $step
+  
+    do_execsql_test 3.3 {
+      SELECT * FROM rt;
+    } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0}
+  
+    do_execsql_test 3.4.1 {
+      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
+    } {1 2}
+    do_execsql_test 3.4.2 {
+      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
+    } {2 3}
+
+    do_rbu_vacuum_test 3.5 $step
+
+    do_execsql_test 3.6.1 {
+      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
+    } {1 2}
+    do_execsql_test 3.6.2 {
+      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
+    } {2 3}
+  }
+
+  ifcapable trigger {
+    reset_db
+    do_execsql_test 4.1 {
+      CREATE TABLE t1(a, b, c);
+      INSERT INTO t1 VALUES(1, 2, 3);
+      CREATE VIEW v1 AS SELECT * FROM t1;
+      CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END;
+    }
+
+    do_execsql_test 4.2 {
+      SELECT * FROM sqlite_master;
+    } {
+    table t1 t1 2 {CREATE TABLE t1(a, b, c)}
+    view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
+    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
+    }
+
+    do_rbu_vacuum_test 4.3 $step
+    do_execsql_test 4.4 {
+      SELECT * FROM sqlite_master;
+    } {
+    table t1 t1 2 {CREATE TABLE t1(a, b, c)}
+    view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
+    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
+    }
+  }
+
+}
+
+finish_test
@@ -921,12 +921,14 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){
  int rc;
  memset(pIter, 0, sizeof(RbuObjIter));

-  rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, 
+  rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, 
+    sqlite3_mprintf(
      "SELECT rbu_target_name(name, type='view') AS target, name "
      "FROM sqlite_master "
      "WHERE type IN ('table', 'view') AND target IS NOT NULL "
+      " %s "
      "ORDER BY name"
-  );
+  , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : ""));

  if( rc==SQLITE_OK ){
    rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg,
@@ -2505,9 +2507,9 @@ static void rbuFileSuffix3(const char *zBase, char *z){
 #endif
  {
    int i, sz;
-    sz = sqlite3Strlen30(z);
+    sz = (int)strlen(z)&0xffffff;
    for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
-    if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4);
+    if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4);
  }
 #endif
 }
@@ -3553,30 +3555,7 @@ static sqlite3rbu *openRbuHandle(
    if( p->rc==SQLITE_OK ){
      if( p->eStage==RBU_STAGE_OAL ){
        sqlite3 *db = p->dbMain;
-
-        if( pState->eStage==0 && rbuIsVacuum(p) ){
-          rbuCopyPragma(p, "page_size");
-          rbuCopyPragma(p, "auto_vacuum");
-        }
-
-        /* Open transactions both databases. The *-oal file is opened or
-        ** created at this point. */
-        if( p->rc==SQLITE_OK ){
-          p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
-        }
-        if( p->rc==SQLITE_OK ){
-          p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg);
-        }
-
-        /* Check if the main database is a zipvfs db. If it is, set the upper
-        ** level pager to use "journal_mode=off". This prevents it from 
-        ** generating a large journal using a temp file.  */
-        if( p->rc==SQLITE_OK ){
-          int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0);
-          if( frc==SQLITE_OK ){
-            p->rc = sqlite3_exec(db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg);
-          }
-        }
+        p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg);

        /* Point the object iterator at the first object */
        if( p->rc==SQLITE_OK ){
@@ -3587,12 +3566,34 @@ static sqlite3rbu *openRbuHandle(
        ** update finished.  */
        if( p->rc==SQLITE_OK && p->objiter.zTbl==0 ){
          p->rc = SQLITE_DONE;
-        }
+          p->eStage = RBU_STAGE_DONE;
+        }else{
+          if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){
+            rbuCopyPragma(p, "page_size");
+            rbuCopyPragma(p, "auto_vacuum");
+          }

-        if( p->rc==SQLITE_OK ){
-          rbuSetupOal(p, pState);
-        }
+          /* Open transactions both databases. The *-oal file is opened or
+          ** created at this point. */
+          if( p->rc==SQLITE_OK ){
+            p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
+          }

+          /* Check if the main database is a zipvfs db. If it is, set the upper
+          ** level pager to use "journal_mode=off". This prevents it from 
+          ** generating a large journal using a temp file.  */
+          if( p->rc==SQLITE_OK ){
+            int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0);
+            if( frc==SQLITE_OK ){
+              p->rc = sqlite3_exec(
+                db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg);
+            }
+          }
+
+          if( p->rc==SQLITE_OK ){
+            rbuSetupOal(p, pState);
+          }
+        }
      }else if( p->eStage==RBU_STAGE_MOVE ){
        /* no-op */
      }else if( p->eStage==RBU_STAGE_CKPT ){
@@ -3759,9 +3760,39 @@ void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
  }
 }

+/*
+** Return the current state of the RBU vacuum or update operation.
+*/
+int sqlite3rbu_state(sqlite3rbu *p){
+  int aRes[] = {
+    0, SQLITE_RBU_STATE_OAL, SQLITE_RBU_STATE_MOVE,
+    0, SQLITE_RBU_STATE_CHECKPOINT, SQLITE_RBU_STATE_DONE
+  };
+
+  assert( RBU_STAGE_OAL==1 );
+  assert( RBU_STAGE_MOVE==2 );
+  assert( RBU_STAGE_CKPT==4 );
+  assert( RBU_STAGE_DONE==5 );
+  assert( aRes[RBU_STAGE_OAL]==SQLITE_RBU_STATE_OAL );
+  assert( aRes[RBU_STAGE_MOVE]==SQLITE_RBU_STATE_MOVE );
+  assert( aRes[RBU_STAGE_CKPT]==SQLITE_RBU_STATE_CHECKPOINT );
+  assert( aRes[RBU_STAGE_DONE]==SQLITE_RBU_STATE_DONE );
+
+  if( p->rc!=SQLITE_OK && p->rc!=SQLITE_DONE ){
+    return SQLITE_RBU_STATE_ERROR;
+  }else{
+    assert( p->rc!=SQLITE_DONE || p->eStage==RBU_STAGE_DONE );
+    assert( p->eStage==RBU_STAGE_OAL
+         || p->eStage==RBU_STAGE_MOVE
+         || p->eStage==RBU_STAGE_CKPT
+         || p->eStage==RBU_STAGE_DONE
+    );
+    return aRes[p->eStage];
+  }
+}
+
 int sqlite3rbu_savestate(sqlite3rbu *p){
  int rc = p->rc;
-  
  if( rc==SQLITE_DONE ) return SQLITE_OK;

  assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE );
@@ -474,6 +474,44 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu);
 */
 void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo);

+/*
+** Obtain an indication as to the current stage of an RBU update or vacuum.
+** This function always returns one of the SQLITE_RBU_STATE_XXX constants
+** defined in this file. Return values should be interpreted as follows:
+**
+** SQLITE_RBU_STATE_OAL:
+**   RBU is currently building a *-oal file. The next call to sqlite3rbu_step()
+**   may either add further data to the *-oal file, or compute data that will
+**   be added by a subsequent call.
+**
+** SQLITE_RBU_STATE_MOVE:
+**   RBU has finished building the *-oal file. The next call to sqlite3rbu_step()
+**   will move the *-oal file to the equivalent *-wal path. If the current
+**   operation is an RBU update, then the updated version of the database
+**   file will become visible to ordinary SQLite clients following the next
+**   call to sqlite3rbu_step().
+**
+** SQLITE_RBU_STATE_CHECKPOINT:
+**   RBU is currently performing an incremental checkpoint. The next call to
+**   sqlite3rbu_step() will copy a page of data from the *-wal file into
+**   the target database file.
+**
+** SQLITE_RBU_STATE_DONE:
+**   The RBU operation has finished. Any subsequent calls to sqlite3rbu_step()
+**   will immediately return SQLITE_DONE.
+**
+** SQLITE_RBU_STATE_ERROR:
+**   An error has occurred. Any subsequent calls to sqlite3rbu_step() will
+**   immediately return the SQLite error code associated with the error.
+*/
+#define SQLITE_RBU_STATE_OAL        1
+#define SQLITE_RBU_STATE_MOVE       2
+#define SQLITE_RBU_STATE_CHECKPOINT 3
+#define SQLITE_RBU_STATE_DONE       4
+#define SQLITE_RBU_STATE_ERROR      5
+
+int sqlite3rbu_state(sqlite3rbu *pRbu);
+
 /*
 ** Create an RBU VFS named zName that accesses the underlying file-system
 ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, 
@@ -69,6 +69,8 @@ static int test_sqlite3rbu_cmd(
    {"dbMain_eval", 3, "SQL"},    /* 4 */
    {"bp_progress", 2, ""},       /* 5 */
    {"db", 3, "RBU"},             /* 6 */
+    {"state", 2, ""},             /* 7 */
+    {"progress", 2, ""},          /* 8 */
    {0,0,0}
  };
  int iCmd;
@@ -166,6 +168,18 @@ static int test_sqlite3rbu_cmd(
      }
      break;
    }
+    case 7: /* state */ {
+      const char *aRes[] = { 0, "oal", "move", "checkpoint", "done", "error" };
+      int eState = sqlite3rbu_state(pRbu);
+      assert( eState>0 && eState<=5 );
+      Tcl_SetResult(interp, (char*)aRes[eState], TCL_STATIC);
+      break;
+    }
+    case 8: /* progress */ {
+      sqlite3_int64 nStep =  sqlite3rbu_progress(pRbu);
+      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nStep));
+      break;
+    }

    default: /* seems unlikely */
      assert( !"cannot happen" );
@@ -325,6 +325,7 @@ TESTSRC = \
 TESTSRC += \
  $(TOP)/ext/misc/amatch.c \
  $(TOP)/ext/misc/closure.c \
+  $(TOP)/ext/misc/csv.c \
  $(TOP)/ext/misc/eval.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
@@ -450,7 +451,8 @@ TESTPROGS = \
  testfixture$(EXE) \
  sqlite3$(EXE) \
  sqlite3_analyzer$(EXE) \
-  sqldiff$(EXE)
+  sqldiff$(EXE) \
+  dbhash$(EXE)

 # Databases containing fuzzer test cases
 #
@@ -487,6 +489,10 @@ sqldiff$(EXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
 	$(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \
 		$(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB)

+dbhash$(EXE):	$(TOP)/tool/dbhash.c sqlite3.c sqlite3.h
+	$(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \
+		$(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB)
+
 scrub$(EXE):	$(TOP)/ext/misc/scrub.c sqlite3.o
 	$(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB)

@@ -1,8 +1,8 @@
-C Enhance\sthe\ssqlite3_load_extension()\sAPI\sso\sthat\sthe\sfirst\sparameter\s(the\s"db"\nparameter)\scan\sbe\sNULL.\s\sAn\sextension\sthat\sis\snot\sassociated\swith\sany\s\ndatabase\sconnection\sremains\sloaded\sfor\sthe\slife\sof\sthe\sprocess.
-D 2016-05-28T15:03:06.118
-F Makefile.in f59e0763ff448719fc1bd25513882b0567286317
+C Add\sthe\ssqlite3rbu_state()\sAPI.\sUsed\sto\sdetermine\sthe\scurrent\sstate\s(creating\sOAL,\sready\sto\smove\sOAL,\sincremental-checkpoint,\sfinished\sor\serror)\sof\san\sRBU\soperation.
+D 2016-07-01T12:39:58.096
+F Makefile.in bc2b4864a23a4a21c3e26d7b4350f51bab324d45
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 306d73e854b1a92ea06e5d1e637faa5c44de53c7
+F Makefile.msc 50149765ef72f4e652b9a0f1f6462c4784bb9423
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
 F VERSION cb29eb11e493dd85b3eeec4053c03949bf98478e
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -14,7 +14,7 @@ F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4
 F autoconf/Makefile.msc 0eca137c12542bd76c253a2d24380f29ade59b95
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
-F autoconf/configure.ac 7754404e955900cfdeba08862cdfa802eca5b0af
+F autoconf/configure.ac cacf2616abf6e4a569bde2ef365c143caeec40bc
 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd
 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873
 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43
@@ -98,14 +98,14 @@ F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h b770c5e0a8d2ee071ddffc7ab722dbf3474a8abe
-F ext/fts5/fts5Int.h 3677076aecbf645a7f2a019115c6a4ec3272dd78
+F ext/fts5/fts5Int.h 9bd0c7c64285b5b368eca0ac63613185c5ad24ba
 F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
-F ext/fts5/fts5_expr.c da2b33c2aac91e96641c0a7cf2bbaa36eb2667f7
+F ext/fts5/fts5_expr.c bcb238ee4ac1164302ab528487520488516bd030
 F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2
 F ext/fts5/fts5_index.c b429e23fabb57506f71e406997cc46b89190dc97
-F ext/fts5/fts5_main.c b4a0fc5bf17f2f1f056ee76cdd7d2af08b360f55
+F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2
 F ext/fts5/fts5_storage.c 3309c6a8e34b974513016fd1ef47c83f5898f94c
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
 F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc
@@ -176,7 +176,7 @@ F ext/fts5/test/fts5porter.test 7cdc07bef301d70eebbfa75dcaf45c3680e1d0e1
 F ext/fts5/test/fts5porter2.test 2e65633d58a1c525d5af0f6c01e5a59155bb3487
 F ext/fts5/test/fts5prefix.test efd42e00bb8e8a36383f25c838185508681c093f
 F ext/fts5/test/fts5query.test f5ec25f5f2fbb70033424113cdffc101b1985a40
-F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab
+F ext/fts5/test/fts5rank.test 3e55e7eeb4c98728e4a3171c9e994e1a2f24eb99
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6
@@ -206,6 +206,7 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
 F ext/misc/amatch.c 211108e201105e4bb0c076527b8cfd34330fc234
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
+F ext/misc/csv.c f01126ba170fd4ef7c752b156568a80c912d4441
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
@@ -221,18 +222,18 @@ F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c bf1b922c2750698e9a3d4c50cce6974adb7e93be
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
-F ext/misc/vfsstat.c 318bb69270291bfff3d5bb58b46397ed7b507589
+F ext/misc/vfsstat.c 6110aeeaab2f1df17a923c8a8acef3c74f6dc515
 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
 F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4
-F ext/rbu/rbu1.test 42bd835e019eff789ec241017965277baeb658b1
+F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 046b0980041d30700464a800bbf6733ed2df515d
 F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702
 F ext/rbu/rbu12.test bde22ed0004dd5d1888c72a84ae407e574aeae16
 F ext/rbu/rbu13.test 462ff799c4afedc3ef8a47ff818c0ffbf14ae4f2
 F ext/rbu/rbu14.test 01f5dcba904aecadbaea69d4ccdc2ea43dd30560
 F ext/rbu/rbu3.test 8bd4c6b87367c358981b6a47dc3d654fa60bff90
-F ext/rbu/rbu5.test 2e24fee3e615aecd99bbdd46967935a641e866f7
+F ext/rbu/rbu5.test cf1fa84d6107efd73e292a7be8e1fb31953ca45c
 F ext/rbu/rbu6.test 32e8ed60631b6facdb6366bd2b5f5f25245e7edb
 F ext/rbu/rbu7.test fd025d5ba440fcfe151fbb0e3835e1e7fe964fa1
 F ext/rbu/rbu8.test 3bbf2c35d71a843c463efe93946f14ad10c3ede0
@@ -240,19 +241,20 @@ F ext/rbu/rbu9.test 0806d1772c9f4981774ff028de6656e4183082af
 F ext/rbu/rbuA.test c1a7b3e2d926b8f8448bb3b4ae787e314ee4b2b3
 F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2
 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831
-F ext/rbu/rbu_common.tcl 0398545fed614f807d5f0ba55a85a51f08ba8f1a
+F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead
 F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695
-F ext/rbu/rbudiff.test 2df0a8a7d998ecf81764c21eeda3cde5611c5091
+F ext/rbu/rbudiff.test 4c9f8df6f723f553781d3d117501b7e9d170a145
 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89
 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca
 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda
 F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a
 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
-F ext/rbu/rbuvacuum.test 66e02cf299836770e718e95c36686be0b26dbda3
-F ext/rbu/sqlite3rbu.c bf36625990c6865ecf08bd844d8097ed2d0a6958
-F ext/rbu/sqlite3rbu.h 2acd0a6344a6079de15c8dc9d84d3df83a665930
-F ext/rbu/test_rbu.c 9bbdf6bd8efd58fbc4f192698df50569598fbb9e
+F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0
+F ext/rbu/rbuvacuum2.test 45009e127c3fb385e5c0fd5a8a63fb922a79d0ab
+F ext/rbu/sqlite3rbu.c 948677ee0ec57da51148e6c5f64ac68afcf36ab2
+F ext/rbu/sqlite3rbu.h db8858120c9be14b60c9225f9da28221f5f6b945
+F ext/rbu/test_rbu.c 9f043b74c46c45b231f4313aed1fccb379a76fe6
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c d26a815b0df1c412a6881dae8d7fd3c9c08cce68
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -305,7 +307,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk eab7db27ac2a32b1172a85aa850c0b9c3a00524c
+F main.mk 3f669c06db5c4a53ff21dda639247c6310497180
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -324,27 +326,27 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
 F src/backup.c 6df65fdd569c901a418887a1a76f82ec35044556
 F src/bitvec.c 3ee4c8b2c94ed3a7377256e18199e6ff5cf33f63
 F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73
-F src/btree.c 2128172fc1c420a6fa6878827fa595407795069a
-F src/btree.h 1342a9b2cc2089e3534d3ef00204786783f6aea6
+F src/btree.c 0621247619df15790f7ed5f14926cfb3925c10b1
+F src/btree.h 2107a2630e02c8cba58bb12ce14e731e734ea29c
 F src/btreeInt.h c18b7d2a3494695133e4e60ee36061d37f45d9a5
-F src/build.c 785fa789319d93c6ae20efbd01d4da9ce8f8a793
+F src/build.c d1fdfd7ab8f5447e494ef15825973bf0719527c6
 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
-F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198
+F src/ctime.c 61949e83c4c36e37195a8398ebc752780b534d95
 F src/date.c 1cc9fb516ec9932c6fd4d2a0d2f8bc4480145c39
-F src/dbstat.c c845548d4346e606e2f2b7d2e714ace2b8a7dd1b
+F src/dbstat.c 4f6f7f52b49beb9636ffbd517cfe44a402ba4ad0
 F src/delete.c 4aba4214a377ce8ddde2d2e609777bcc8235200f
-F src/expr.c 81bd7d87985746313770211183e900ed1ad28381
+F src/expr.c 523a5b1db2b6d88c6eefb224877bf635a3bcfc92
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
-F src/fkey.c 4c0bd09e602b8ae8d36d81e31e4872d0b53c87bb
-F src/func.c ef4c18c8a66143413ce41a58d582d2c14ddf78e1
+F src/fkey.c bc4145347595b7770f9a598cff1c848302cf5413
+F src/func.c 61a4114cf7004f10c542cfabbab9f2bcb9033045
 F src/global.c c45ea22aff29334f6a9ec549235ac3357c970015
 F src/hash.c 55b5fb474100cee0b901edaf203e26c970940f36
 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
 F src/insert.c 8f4e9fcbd8e95e85f15647ba8b413b18d556ec2b
 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e
-F src/loadext.c f2316d5ccd905d86260392fb8d2b5fd67335ed77
+F src/loadext.c 4237fd37ca589f1d90b3ea925dd9df20da8e0c84
 F src/main.c 405d13e3a4f7c5add9fb27702ae70ed0a6e32cca
 F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -365,34 +367,34 @@ F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
 F src/os_unix.c a9443cdab41d7f3cdf0df3a5aab62fd6e1c9b234
-F src/os_win.c 852fc2ff6084296348ed3739c548b2cf32df394e
+F src/os_win.c d4b8faf8896b65818e67070711fdd00d8e620bd6
 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
 F src/pager.c c368634b888b1c8740aea83b36bfd266f2443e60
 F src/pager.h 8ab6b6feeee4bc0439bfde7ee59ba99df98b9bc3
-F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e
+F src/parse.y 01b9f37c4c7009ab56fda98bc7db4c42643cecfe
 F src/pcache.c 5583c8ade4b05075a60ba953ef471d1c1a9c05df
 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490
 F src/pcache1.c 7f51d2b541aab57596adf62db2c4bb025d34f04d
-F src/pragma.c d98039af2deb5a4990f6635462b3f61f325a6c75
+F src/pragma.c c8b499756658cb8b82cfdbb5845c22cf11f297aa
 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c
 F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e
 F src/printf.c a5f0ca08ddede803c241266abb46356ec748ded1
 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696
 F src/resolve.c cca3aa77b95706df5d635a2141a4d1de60ae6598
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c a0c4abf54bc6bd3a9c77a36ef3d1676045706cb2
-F src/shell.c c80b8d562b66d27172b0e98333f2f27876d8b4aa
-F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54
+F src/select.c f3c6e9065fb34f6a23af27ec7f1f717ffbfc2ee4
+F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6
+F src/sqlite.h.in 5f8113dbec74c6c093ead9930afb8c9fbd9f643d
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 98f72cbfe00169c39089115427d06ea05fe4b4a2
-F src/sqliteInt.h 09621b4b7dba808b24262c2480ea75b045001853
+F src/sqliteInt.h 2ac73e9fdd138f4607139d9bd6a1e6dcd01837dc
 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247
 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9
-F src/tclsqlite.c 9c4c4589d078de37813ded708d8838b338ffb060
-F src/test1.c c0e5b69f99e95a2c9f55fdb6e46b44f1a15292d8
+F src/tclsqlite.c 25fbbbb97f76dbfd113153fb63f52d7ecfac5dd0
+F src/test1.c 43b37ab2b7338fd3313e74902f0d6c821eae843b
 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b
-F src/test3.c d2c9efd2985ff8f5502ffd3253156984778d77d8
+F src/test3.c c75c8af0eadb335236c9e61b51044c58a8f7dd59
 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e
 F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1
 F src/test6.c a684b7abd01352ab50cb79c0bf727e6b3f381a3d
@@ -446,24 +448,24 @@ F src/update.c 4f05ea8cddfa367d045e03589756c02199e8f9bd
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c 810ec3f22e2d1b62e66c30fe3621ebdedd23584d
 F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52
-F src/vdbe.c 45e4da739187028281e342f79fae0f4145055bec
+F src/vdbe.c 22b46c3b725e950e9f2760e2d76953d592600ad4
 F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170
 F src/vdbeInt.h ddb157974436d87652de7dc641f7191496d9a8cd
-F src/vdbeapi.c ba85b78fe08dc4a9ce747e62c89a2b4a4547e74c
-F src/vdbeaux.c 1d6b9a979d1036db7bc39990e9e683f19520bc5c
-F src/vdbeblob.c c9f2f494b911c6fa34efd9803f0a10807da80f77
-F src/vdbemem.c 5cfef60e60e19cab6275d1b975bf4c791d575beb
+F src/vdbeapi.c 02bcbc2ca5d2004b029088b05b468b394881e103
+F src/vdbeaux.c c90275b0e55a2b32c03dc09314194fe46f2429d8
+F src/vdbeblob.c 83d2d266383157b02e2b809350bb197e89d7895b
+F src/vdbemem.c 1ecaa5ee0caff07255f25d04e8dc88befb6f88d1
 F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c
 F src/vdbetrace.c f75c5455d8cf389ef86a8bfdfd3177e0e3692484
-F src/vtab.c ce0f2ebb589b459b32c640b33af64bfa5b29aaf8
+F src/vtab.c 948d2d4984219eee37a7bf427d6667e21e6eb92e
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 02eeecc265f6ffd0597378f5d8ae9070b62a406a
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c b9f5b0ddb14c3827e70b5379e659cf4cfd524c4d
+F src/where.c 48eed8ebe319c6cbc7bf7682018f32af0f5189f5
 F src/whereInt.h e5b939701a7ceffc5a3a8188a37f9746416ebcd0
-F src/wherecode.c ba71a4e4bada29aa9842200e6299714bf18c812c
-F src/whereexpr.c eacc0e60d029a082b4fc0cc42ea98544add1319e
+F src/wherecode.c e20cb381ff621e56a4684c71e31999aca2547ca6
+F src/whereexpr.c c32d47085dbaca0b8fd013210f56693c7d220d48
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -526,6 +528,7 @@ F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f
 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c
 F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee
 F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060
+F test/bestindex3.test b80da904d23581d233a7ceee7d6bbad2b23a8133
 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c
 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59
 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc
@@ -611,8 +614,10 @@ F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d
 F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2
 F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8
 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c
-F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47
+F test/csv01.test 0929a9ce47021519512be92861f29e32d2538e5f
+F test/ctime.test 0b995accd44a52914bd4744d5c1b6e1a56c7897c
 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856
+F test/cursorhint2.test 2b45341d32d1aae9991a00ef31ebca339b274255
 F test/date.test 984ac1e3e5e031386866f034006148d3972b4a65
 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204
 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab
@@ -836,7 +841,7 @@ F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407
 F test/index3.test 81bc47890b8abfb181bc35f8d10b56c069803386
 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6
 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7
-F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a
+F test/index6.test 43b4e29258b978fcdab84fc61df4f5212119dd09
 F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c
 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985
 F test/indexexpr1.test cb71b6586177b840e28110dd952178bb2bdfedc2
@@ -1036,7 +1041,7 @@ F test/select6.test 39eac4a5c03650b2b473c532882273283ee8b7a0
 F test/select7.test 95e370c42d47c3c52377d05e9ffc01ccff7c1f61
 F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d
 F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95
-F test/selectA.test e452bdb975f488ea46d091382a9185b5853ed2c7
+F test/selectA.test 101e722370ac6e84978c2958b8931c78b10a1709
 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25
 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977
 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394
@@ -1089,14 +1094,14 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/speedtest1.c 4476f7030775507f060d27b789e1f4123c5a5866
+F test/speedtest1.c 28221f433d358dd3fcf2ca504ed51409aa66ffaf
 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e
 F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142
 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a
-F test/stat.test 66e95f97b9f724f9ab921d054ee0db3c2689f1ee
+F test/stat.test ab95d28503d0f6d98ffd8ce204643c9da090ebf1
 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1
 F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f
 F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f
@@ -1113,7 +1118,7 @@ F test/tabfunc01.test f977868fa8bb7beb4b2072883190411653473906
 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f
 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126
 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930
-F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa
+F test/tclsqlite.test cf0d0a3fd03d64892cec2d48aae9fb2f148680a5
 F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6
 F test/tempdb2.test 4fc92055f2a3f7626c0f2eabbb637dc021b311d5
 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900
@@ -1357,7 +1362,7 @@ F test/walcksum.test bb234a1bb42248b3515d992b719708015c384278
 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a
 F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36
 F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af
-F test/walcrash4.test fa7d92ab84fe3ff409e1cda88a622545e2bc2b66
+F test/walcrash4.test 3374d6a0813bfe9d841a973958e0552b545a6423
 F test/walfault.test 1f8389f7709877e9b4cc679033d71d6fe529056b
 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483
 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c
@@ -1371,7 +1376,7 @@ F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
 F test/where.test f0c325563acde44f2c4ea6ba348e9e29f7121757
 F test/where2.test 478d2170637b9211f593120648858593bf2445a1
-F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e
+F test/where3.test 54cdeb02157acc979de41530b804ae7b09552bf1
 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8
 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b
@@ -1416,6 +1421,7 @@ F tool/build-all-msvc.bat 3e4e4043b53f1aede4308e0d2567bbd773614630 x
 F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
 F tool/cg_anno.tcl 692ce4b8693d59e3a3de77ca97f4139ecfa641b0 x
 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2
+F tool/dbhash.c a06228aa21ebc4e6ea8daa486601d938499238a5
 F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2
 F tool/fast_vacuum.c 5ba0d6f5963a0a63bdc42840f678bad75b2ebce1
 F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439
@@ -1424,7 +1430,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/lemon.c 09a96bed19955697a5e20c49ad863ec2005815a2
-F tool/lempar.c f06b7e98a6b7efb404375b6f4a0c71c85faa1512
+F tool/lempar.c 8c4e9d8517e50da391f1d89a519e743dd4afbc09
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1463,7 +1469,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
-F tool/sqldiff.c ca315aca4e2d24233e8f2000edea5880c53d1875
+F tool/sqldiff.c 131e9c6e09d3d78e3c4ae8b5fd23451521723aac
 F tool/srcck1.c 4c39bdfa9a92edd20233ee720df84dbeb2417602
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1496,10 +1502,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0987487dd4ebfcf66ddeec8ceca47775216a0887
-R e12f783957b755fbf23064ac37b7305b
-T *branch * persistent-extensions
-T *sym-persistent-extensions *
+P f81050859170c8708a1b296da8dd3ef0dd314a11
+R 2c74d9aac55f1cbb7c7d3232cf9071e1
+T *branch * rbu-state-api
+T *sym-rbu-state-api *
 T -sym-trunk *
-U drh
-Z 7c91f0eea3a925fa49de1662b44330b3
+U dan
+Z b7514d1a78fc46b36ed5446f76a12e50
@@ -1 +1 @@
-3a461043898a77b14ffc12b5e332b3d65a8dca1d
+92e7df0ff5c4c118c63d92a767dc82700438a310
@@ -450,6 +450,15 @@ static void releasePage(MemPage *pPage);  /* Forward reference */
 static int cursorHoldsMutex(BtCursor *p){
  return sqlite3_mutex_held(p->pBt->mutex);
 }
+
+/* Verify that the cursor and the BtShared agree about what is the current
+** database connetion. This is important in shared-cache mode. If the database 
+** connection pointers get out-of-sync, it is possible for routines like
+** btreeInitPage() to reference an stale connection pointer that references a
+** a connection that has already closed.  This routine is used inside assert()
+** statements only and for the purpose of double-checking that the btree code
+** does keep the database connection pointers up-to-date.
+*/
 static int cursorOwnsBtShared(BtCursor *p){
  assert( cursorHoldsMutex(p) );
  return (p->pBtree->db==p->pBt->db);
@@ -609,21 +618,19 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){
 ** the key.
 */
 static int saveCursorKey(BtCursor *pCur){
-  int rc;
+  int rc = SQLITE_OK;
  assert( CURSOR_VALID==pCur->eState );
  assert( 0==pCur->pKey );
  assert( cursorHoldsMutex(pCur) );

-  rc = sqlite3BtreeKeySize(pCur, &pCur->nKey);
-  assert( rc==SQLITE_OK );  /* KeySize() cannot fail */
-
-  /* If this is an intKey table, then the above call to BtreeKeySize()
-  ** stores the integer key in pCur->nKey. In this case this value is
-  ** all that is required. Otherwise, if pCur is not open on an intKey
-  ** table, then malloc space for and store the pCur->nKey bytes of key 
-  ** data.  */
-  if( 0==pCur->curIntKey ){
-    void *pKey = sqlite3Malloc( pCur->nKey );
+  if( pCur->curIntKey ){
+    /* Only the rowid is required for a table btree */
+    pCur->nKey = sqlite3BtreeIntegerKey(pCur);
+  }else{
+    /* For an index btree, save the complete key content */
+    void *pKey;
+    pCur->nKey = sqlite3BtreePayloadSize(pCur);
+    pKey = sqlite3Malloc( pCur->nKey );
    if( pKey ){
      rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey);
      if( rc==SQLITE_OK ){
@@ -4262,46 +4269,33 @@ int sqlite3BtreeCursorIsValid(BtCursor *pCur){
 #endif /* NDEBUG */

 /*
-** Set *pSize to the size of the buffer needed to hold the value of
-** the key for the current entry.  If the cursor is not pointing
-** to a valid entry, *pSize is set to 0. 
-**
-** For a table with the INTKEY flag set, this routine returns the key
-** itself, not the number of bytes in the key.
-**
-** The caller must position the cursor prior to invoking this routine.
-** 
-** This routine cannot fail.  It always returns SQLITE_OK.  
+** Return the value of the integer key or "rowid" for a table btree.
+** This routine is only valid for a cursor that is pointing into a
+** ordinary table btree.  If the cursor points to an index btree or
+** is invalid, the result of this routine is undefined.
 */
-int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){
+i64 sqlite3BtreeIntegerKey(BtCursor *pCur){
  assert( cursorHoldsMutex(pCur) );
  assert( pCur->eState==CURSOR_VALID );
+  assert( pCur->curIntKey );
  getCellInfo(pCur);
-  *pSize = pCur->info.nKey;
-  return SQLITE_OK;
+  return pCur->info.nKey;
 }

 /*
-** Set *pSize to the number of bytes of data in the entry the
-** cursor currently points to.
+** Return the number of bytes of payload for the entry that pCur is
+** currently pointing to.  For table btrees, this will be the amount
+** of data.  For index btrees, this will be the size of the key.
 **
 ** The caller must guarantee that the cursor is pointing to a non-NULL
 ** valid entry.  In other words, the calling procedure must guarantee
 ** that the cursor has Cursor.eState==CURSOR_VALID.
-**
-** Failure is not possible.  This function always returns SQLITE_OK.
-** It might just as well be a procedure (returning void) but we continue
-** to return an integer result code for historical reasons.
 */
-int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){
-  assert( cursorOwnsBtShared(pCur) );
+u32 sqlite3BtreePayloadSize(BtCursor *pCur){
+  assert( cursorHoldsMutex(pCur) );
  assert( pCur->eState==CURSOR_VALID );
-  assert( pCur->iPage>=0 );
-  assert( pCur->iPage<BTCURSOR_MAX_DEPTH );
-  assert( pCur->apPage[pCur->iPage]->intKeyLeaf==1 );
  getCellInfo(pCur);
-  *pSize = pCur->info.nPayload;
-  return SQLITE_OK;
+  return pCur->info.nPayload;
 }

 /*
@@ -4743,10 +4737,7 @@ static const void *fetchPayload(
 ** These routines is used to get quick access to key and data
 ** in the common case where no overflow pages are used.
 */
-const void *sqlite3BtreeKeyFetch(BtCursor *pCur, u32 *pAmt){
-  return fetchPayload(pCur, pAmt);
-}
-const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){
+const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){
  return fetchPayload(pCur, pAmt);
 }

@@ -5079,11 +5070,12 @@ int sqlite3BtreeMovetoUnpacked(
  assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
  assert( pRes );
  assert( (pIdxKey==0)==(pCur->pKeyInfo==0) );
+  assert( pCur->eState!=CURSOR_VALID || (pIdxKey==0)==(pCur->curIntKey!=0) );

  /* If the cursor is already positioned at the point we are trying
  ** to move to, then just return without doing any work */
-  if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0
-   && pCur->curIntKey 
+  if( pIdxKey==0
+   && pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0
  ){
    if( pCur->info.nKey==intKey ){
      *pRes = 0;
@@ -284,11 +284,10 @@ int sqlite3BtreeLast(BtCursor*, int *pRes);
 int sqlite3BtreeNext(BtCursor*, int *pRes);
 int sqlite3BtreeEof(BtCursor*);
 int sqlite3BtreePrevious(BtCursor*, int *pRes);
-int sqlite3BtreeKeySize(BtCursor*, i64 *pSize);
+i64 sqlite3BtreeIntegerKey(BtCursor*);
 int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*);
-const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt);
-const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt);
-int sqlite3BtreeDataSize(BtCursor*, u32 *pSize);
+const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt);
+u32 sqlite3BtreePayloadSize(BtCursor*);
 int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*);

 char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*);
@@ -338,7 +338,7 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){
 */
 Table *sqlite3LocateTable(
  Parse *pParse,         /* context in which to report errors */
-  int isView,            /* True if looking for a VIEW rather than a TABLE */
+  u32 flags,             /* LOCATE_VIEW or LOCATE_NOERR */
  const char *zName,     /* Name of the table we are looking for */
  const char *zDbase     /* Name of the database.  Might be NULL */
 ){
@@ -352,7 +352,7 @@ Table *sqlite3LocateTable(

  p = sqlite3FindTable(pParse->db, zName, zDbase);
  if( p==0 ){
-    const char *zMsg = isView ? "no such view" : "no such table";
+    const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table";
 #ifndef SQLITE_OMIT_VIRTUALTABLE
    if( sqlite3FindDbName(pParse->db, zDbase)<1 ){
      /* If zName is the not the name of a table in the schema created using
@@ -364,12 +364,14 @@ Table *sqlite3LocateTable(
      }
    }
 #endif
-    if( zDbase ){
-      sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName);
-    }else{
-      sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName);
+    if( (flags & LOCATE_NOERR)==0 ){
+      if( zDbase ){
+        sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName);
+      }else{
+        sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName);
+      }
+      pParse->checkSchema = 1;
    }
-    pParse->checkSchema = 1;
  }

  return p;
@@ -386,7 +388,7 @@ Table *sqlite3LocateTable(
 */
 Table *sqlite3LocateTableItem(
  Parse *pParse, 
-  int isView, 
+  u32 flags,
  struct SrcList_item *p
 ){
  const char *zDb;
@@ -397,7 +399,7 @@ Table *sqlite3LocateTableItem(
  }else{
    zDb = p->zDatabase;
  }
-  return sqlite3LocateTable(pParse, isView, p->zName, zDb);
+  return sqlite3LocateTable(pParse, flags, p->zName, zDb);
 }

 /*
@@ -605,8 +607,9 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
  /* Delete all indices associated with this table. */
  for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){
    pNext = pIndex->pNext;
-    assert( pIndex->pSchema==pTable->pSchema );
-    if( !db || db->pnBytesFreed==0 ){
+    assert( pIndex->pSchema==pTable->pSchema
+         || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) );
+    if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){
      char *zName = pIndex->zName; 
      TESTONLY ( Index *pOld = ) sqlite3HashInsert(
         &pIndex->pSchema->idxHash, zName, 0
@@ -1288,7 +1291,7 @@ void sqlite3AddPrimaryKey(
  Column *pCol = 0;
  int iCol = -1, i;
  int nTerm;
-  if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit;
+  if( pTab==0 ) goto primary_key_exit;
  if( pTab->tabFlags & TF_HasPrimaryKey ){
    sqlite3ErrorMsg(pParse, 
      "table \"%s\" has more than one primary key", pTab->zName);
@@ -1334,12 +1337,8 @@ void sqlite3AddPrimaryKey(
       "INTEGER PRIMARY KEY");
 #endif
  }else{
-    Index *p;
-    p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0,
-                           0, sortOrder, 0);
-    if( p ){
-      p->idxType = SQLITE_IDXTYPE_PRIMARYKEY;
-    }
+    sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0,
+                           0, sortOrder, 0, SQLITE_IDXTYPE_PRIMARYKEY);
    pList = 0;
  }

@@ -1656,21 +1655,23 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){
 ** are appropriate for a WITHOUT ROWID table instead of a rowid table.
 ** Changes include:
 **
-**     (1)  Convert the OP_CreateTable into an OP_CreateIndex.  There is
+**     (1)  Set all columns of the PRIMARY KEY schema object to be NOT NULL.
+**     (2)  Convert the OP_CreateTable into an OP_CreateIndex.  There is
 **          no rowid btree for a WITHOUT ROWID.  Instead, the canonical
 **          data storage is a covering index btree.
-**     (2)  Bypass the creation of the sqlite_master table entry
+**     (3)  Bypass the creation of the sqlite_master table entry
 **          for the PRIMARY KEY as the primary key index is now
 **          identified by the sqlite_master table entry of the table itself.
-**     (3)  Set the Index.tnum of the PRIMARY KEY Index object in the
+**     (4)  Set the Index.tnum of the PRIMARY KEY Index object in the
 **          schema to the rootpage from the main table.
-**     (4)  Set all columns of the PRIMARY KEY schema object to be NOT NULL.
 **     (5)  Add all table columns to the PRIMARY KEY Index object
 **          so that the PRIMARY KEY is a covering index.  The surplus
 **          columns are part of KeyInfo.nXField and are not used for
 **          sorting or lookup or uniqueness checks.
 **     (6)  Replace the rowid tail on all automatically generated UNIQUE
 **          indices with the PRIMARY KEY columns.
+**
+** For virtual tables, only (1) is performed.
 */
 static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
  Index *pIdx;
@@ -1680,6 +1681,20 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
  sqlite3 *db = pParse->db;
  Vdbe *v = pParse->pVdbe;

+  /* Mark every PRIMARY KEY column as NOT NULL (except for imposter tables)
+  */
+  if( !db->init.imposterTable ){
+    for(i=0; i<pTab->nCol; i++){
+      if( (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 ){
+        pTab->aCol[i].notNull = OE_Abort;
+      }
+    }
+  }
+
+  /* The remaining transformations only apply to b-tree tables, not to
+  ** virtual tables */
+  if( IN_DECLARE_VTAB ) return;
+
  /* Convert the OP_CreateTable opcode that would normally create the
  ** root-page for the table into an OP_CreateIndex opcode.  The index
  ** created will become the PRIMARY KEY index.
@@ -1701,9 +1716,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
    if( pList==0 ) return;
    pList->a[0].sortOrder = pParse->iPkSortOrder;
    assert( pParse->pNewTable==pTab );
-    pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0);
-    if( pPk==0 ) return;
-    pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY;
+    sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0,
+                       SQLITE_IDXTYPE_PRIMARYKEY);
+    if( db->mallocFailed ) return;
+    pPk = sqlite3PrimaryKeyIndex(pTab);
    pTab->iPKey = -1;
  }else{
    pPk = sqlite3PrimaryKeyIndex(pTab);
@@ -1731,19 +1747,11 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
    }
    pPk->nKeyCol = j;
  }
-  pPk->isCovering = 1;
  assert( pPk!=0 );
+  pPk->isCovering = 1;
+  if( !db->init.imposterTable ) pPk->uniqNotNull = 1;
  nPk = pPk->nKeyCol;

-  /* Make sure every column of the PRIMARY KEY is NOT NULL.  (Except,
-  ** do not enforce this for imposter tables.) */
-  if( !db->init.imposterTable ){
-    for(i=0; i<nPk; i++){
-      pTab->aCol[pPk->aiColumn[i]].notNull = OE_Abort;
-    }
-    pPk->uniqNotNull = 1;
-  }
-
  /* The root page of the PRIMARY KEY is the table root page */
  pPk->tnum = pTab->tnum;

@@ -2498,6 +2506,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){
  assert( pName->nSrc==1 );
  if( sqlite3ReadSchema(pParse) ) goto exit_drop_table;
  if( noErr ) db->suppressErr++;
+  assert( isView==0 || isView==LOCATE_VIEW );
  pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]);
  if( noErr ) db->suppressErr--;

@@ -2868,12 +2877,8 @@ Index *sqlite3AllocateIndexObject(
 ** pList is a list of columns to be indexed.  pList will be NULL if this
 ** is a primary key or unique-constraint on the most recent column added
 ** to the table currently under construction.  
-**
-** If the index is created successfully, return a pointer to the new Index
-** structure. This is used by sqlite3AddPrimaryKey() to mark the index
-** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY)
 */
-Index *sqlite3CreateIndex(
+void sqlite3CreateIndex(
  Parse *pParse,     /* All information about this parse */
  Token *pName1,     /* First part of index name. May be NULL */
  Token *pName2,     /* Second part of index name. May be NULL */
@@ -2883,9 +2888,9 @@ Index *sqlite3CreateIndex(
  Token *pStart,     /* The CREATE token that begins this statement */
  Expr *pPIWhere,    /* WHERE clause for partial indices */
  int sortOrder,     /* Sort order of primary key when pList==NULL */
-  int ifNotExist     /* Omit error if index already exists */
+  int ifNotExist,    /* Omit error if index already exists */
+  u8 idxType         /* The index type */
 ){
-  Index *pRet = 0;     /* Pointer to return */
  Table *pTab = 0;     /* Table to be indexed */
  Index *pIndex = 0;   /* The index to be created */
  char *zName = 0;     /* Name of the index */
@@ -2903,7 +2908,10 @@ Index *sqlite3CreateIndex(
  char *zExtra = 0;                /* Extra space after the Index object */
  Index *pPk = 0;      /* PRIMARY KEY index for WITHOUT ROWID tables */

-  if( db->mallocFailed || IN_DECLARE_VTAB || pParse->nErr>0 ){
+  if( db->mallocFailed || pParse->nErr>0 ){
+    goto exit_create_index;
+  }
+  if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){
    goto exit_create_index;
  }
  if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
@@ -3092,7 +3100,7 @@ Index *sqlite3CreateIndex(
  pIndex->pTable = pTab;
  pIndex->onError = (u8)onError;
  pIndex->uniqNotNull = onError!=OE_None;
-  pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE;
+  pIndex->idxType = idxType;
  pIndex->pSchema = db->aDb[iDb].pSchema;
  pIndex->nKeyCol = pList->nExpr;
  if( pPIWhere ){
@@ -3272,7 +3280,7 @@ Index *sqlite3CreateIndex(
            pIdx->onError = pIndex->onError;
          }
        }
-        pRet = pIdx;
+        if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType;
        goto exit_create_index;
      }
    }
@@ -3284,6 +3292,7 @@ Index *sqlite3CreateIndex(
  assert( pParse->nErr==0 );
  if( db->init.busy ){
    Index *p;
+    assert( !IN_DECLARE_VTAB );
    assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) );
    p = sqlite3HashInsert(&pIndex->pSchema->idxHash, 
                          pIndex->zName, pIndex);
@@ -3390,7 +3399,6 @@ Index *sqlite3CreateIndex(
      pIndex->pNext = pOther->pNext;
      pOther->pNext = pIndex;
    }
-    pRet = pIndex;
    pIndex = 0;
  }

@@ -3401,7 +3409,6 @@ exit_create_index:
  sqlite3ExprListDelete(db, pList);
  sqlite3SrcListDelete(db, pTblName);
  sqlite3DbFree(db, zName);
-  return pRet;
 }

 /*
@@ -3430,10 +3437,11 @@ void sqlite3DefaultRowEst(Index *pIdx){
  int i;

  /* Set the first entry (number of rows in the index) to the estimated 
-  ** number of rows in the table. Or 10, if the estimated number of rows 
-  ** in the table is less than that.  */
+  ** number of rows in the table, or half the number of rows in the table
+  ** for a partial index.   But do not let the estimate drop below 10. */
  a[0] = pIdx->pTable->nRowLogEst;
-  if( a[0]<33 ) a[0] = 33;        assert( 33==sqlite3LogEst(10) );
+  if( pIdx->pPartIdxWhere!=0 ) a[0] -= 10;  assert( 10==sqlite3LogEst(2) );
+  if( a[0]<33 ) a[0] = 33;                  assert( 33==sqlite3LogEst(10) );

  /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is
  ** 6 and each subsequent value (if any) is 5.  */
@@ -4315,10 +4323,6 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){
 /*
 ** Return a KeyInfo structure that is appropriate for the given Index.
 **
-** The KeyInfo structure for an index is cached in the Index object.
-** So there might be multiple references to the returned pointer.  The
-** caller should not try to modify the KeyInfo object.
-**
 ** The caller should invoke sqlite3KeyInfoUnref() on the returned object
 ** when it has finished using it.
 */
@@ -45,6 +45,13 @@ static const char * const azCompileOpt[] = {
 #if SQLITE_CHECK_PAGES
  "CHECK_PAGES",
 #endif
+#if defined(__clang__) && defined(__clang_version__)
+  "COMPILER=clang-" __clang_version__,
+#elif defined(_MSC_VER)
+  "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER),
+#elif defined(__GNUC__) && defined(__VERSION__)
+  "COMPILER=gcc-" __VERSION__,
+#endif
 #if SQLITE_COVERAGE_TEST
  "COVERAGE_TEST",
 #endif
@@ -64,7 +71,7 @@ static const char * const azCompileOpt[] = {
  "DISABLE_LFS",
 #endif
 #if SQLITE_ENABLE_8_3_NAMES
-  "ENABLE_8_3_NAMES",
+  "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES),
 #endif
 #if SQLITE_ENABLE_API_ARMOR
  "ENABLE_API_ARMOR",
@@ -58,10 +58,10 @@
 */
 #define VTAB_SCHEMA                                                         \
  "CREATE TABLE xx( "                                                       \
-  "  name       STRING,           /* Name of table or index */"             \
-  "  path       INTEGER,          /* Path to page from root */"             \
+  "  name       TEXT,             /* Name of table or index */"             \
+  "  path       TEXT,             /* Path to page from root */"             \
  "  pageno     INTEGER,          /* Page number */"                        \
-  "  pagetype   STRING,           /* 'internal', 'leaf' or 'overflow' */"   \
+  "  pagetype   TEXT,             /* 'internal', 'leaf' or 'overflow' */"   \
  "  ncell      INTEGER,          /* Cells on page (0 for overflow) */"     \
  "  payload    INTEGER,          /* Bytes of payload on this page */"      \
  "  unused     INTEGER,          /* Bytes of unused space on this page */" \
@@ -1818,6 +1818,11 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){
          eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0];

          if( prRhsHasNull && !pTab->aCol[iCol].notNull ){
+#ifdef SQLITE_ENABLE_COLUMN_USED_MASK
+            const i64 sOne = 1;
+            sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, 
+                iTab, 0, 0, (u8*)&sOne, P4_INT64);
+#endif
            *prRhsHasNull = ++pParse->nMem;
            sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull);
          }
@@ -2535,7 +2540,7 @@ void sqlite3ExprCodeGetColumnOfTable(
  }else{
    int op = IsVirtual(pTab) ? OP_VColumn : OP_Column;
    int x = iCol;
-    if( !HasRowid(pTab) ){
+    if( !HasRowid(pTab) && !IsVirtual(pTab) ){
      x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol);
    }
    sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut);
@@ -1372,7 +1372,8 @@ void sqlite3FkDelete(sqlite3 *db, Table *pTab){
  FKey *pFKey;                    /* Iterator variable */
  FKey *pNext;                    /* Copy of pFKey->pNextFrom */

-  assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) );
+  assert( db==0 || IsVirtual(pTab)
+         || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) );
  for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){

    /* Remove the FK from the fkeyHash hash table. */
@@ -740,7 +740,7 @@ static int patternCompare(
    }
    c2 = Utf8Read(zString);
    if( c==c2 ) continue;
-    if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){
+    if( noCase  && sqlite3Tolower(c)==sqlite3Tolower(c2) && c<0x80 && c2<0x80 ){
      continue;
    }
    if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue;
@@ -437,7 +437,7 @@ static int sqlite3LoadExtension(
  const char *zProc,    /* Entry point.  Use "sqlite3_extension_init" if 0 */
  char **pzErrMsg       /* Put error message here if not 0 */
 ){
-  sqlite3_vfs *pVfs = db ? db->pVfs : sqlite3_vfs_find(0);
+  sqlite3_vfs *pVfs = db->pVfs;
  void *handle;
  int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*);
  char *zErrmsg = 0;
@@ -446,6 +446,7 @@ static int sqlite3LoadExtension(
  void **aHandle;
  u64 nMsg = 300 + sqlite3Strlen30(zFile);
  int ii;
+  int rc;

  /* Shared library endings to try if zFile cannot be loaded as written */
  static const char *azEndings[] = {
@@ -468,7 +469,7 @@ static int sqlite3LoadExtension(
  ** sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, 1, 0)
  ** to turn on extension loading.
  */
-  if( db && (db->flags & SQLITE_LoadExtension)==0 ){
+  if( (db->flags & SQLITE_LoadExtension)==0 ){
    if( pzErrMsg ){
      *pzErrMsg = sqlite3_mprintf("not authorized");
    }
@@ -548,7 +549,9 @@ static int sqlite3LoadExtension(
    return SQLITE_ERROR;
  }
  sqlite3_free(zAltEntry);
-  if( xInit(db, &zErrmsg, &sqlite3Apis) ){
+  rc = xInit(db, &zErrmsg, &sqlite3Apis);
+  if( rc ){
+    if( rc==SQLITE_OK_LOAD_PERMANENTLY ) return SQLITE_OK;
    if( pzErrMsg ){
      *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg);
    }
@@ -558,18 +561,17 @@ static int sqlite3LoadExtension(
  }

  /* Append the new shared library handle to the db->aExtension array. */
-  if( db ){
-    aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1));
-    if( aHandle==0 ){
-      return SQLITE_NOMEM_BKPT;
-    }
-    if( db->nExtension>0 ){
-      memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension);
-    }
-    sqlite3DbFree(db, db->aExtension);
-    db->aExtension = aHandle;
-    db->aExtension[db->nExtension++] = handle;
+  aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1));
+  if( aHandle==0 ){
+    return SQLITE_NOMEM_BKPT;
  }
+  if( db->nExtension>0 ){
+    memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension);
+  }
+  sqlite3DbFree(db, db->aExtension);
+  db->aExtension = aHandle;
+
+  db->aExtension[db->nExtension++] = handle;
  return SQLITE_OK;
 }
 int sqlite3_load_extension(
@@ -579,7 +581,6 @@ int sqlite3_load_extension(
  char **pzErrMsg       /* Put error message here if not 0 */
 ){
  int rc;
-  if( db==0 ) return sqlite3LoadExtension(0, zFile, zProc, pzErrMsg);
  sqlite3_mutex_enter(db->mutex);
  rc = sqlite3LoadExtension(db, zFile, zProc, pzErrMsg);
  rc = sqlite3ApiExit(db, rc);
@@ -5269,6 +5269,18 @@ static int winFullPathname(
  int nFull,                    /* Size of output buffer in bytes */
  char *zFull                   /* Output buffer */
 ){
+#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__)
+  DWORD nByte;
+  void *zConverted;
+  char *zOut;
+#endif
+
+  /* If this path name begins with "/X:", where "X" is any alphabetic
+  ** character, discard the initial "/" from the pathname.
+  */
+  if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){
+    zRelative++;
+  }

 #if defined(__CYGWIN__)
  SimulateIOError( return SQLITE_ERROR );
@@ -5347,17 +5359,6 @@ static int winFullPathname(
 #endif

 #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__)
-  DWORD nByte;
-  void *zConverted;
-  char *zOut;
-
-  /* If this path name begins with "/X:", where "X" is any alphabetic
-  ** character, discard the initial "/" from the pathname.
-  */
-  if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){
-    zRelative++;
-  }
-
  /* It's odd to simulate an io-error here, but really this is just
  ** using the io-error infrastructure to test that SQLite handles this
  ** function failing. This function could fail if, for example, the
@@ -300,7 +300,8 @@ ccons ::= NULL onconf.
 ccons ::= NOT NULL onconf(R).    {sqlite3AddNotNull(pParse, R);}
 ccons ::= PRIMARY KEY sortorder(Z) onconf(R) autoinc(I).
                                 {sqlite3AddPrimaryKey(pParse,0,R,I,Z);}
-ccons ::= UNIQUE onconf(R).      {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0);}
+ccons ::= UNIQUE onconf(R).      {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0,
+                                   SQLITE_IDXTYPE_UNIQUE);}
 ccons ::= CHECK LP expr(X) RP.   {sqlite3AddCheckConstraint(pParse,X.pExpr);}
 ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R).
                                 {sqlite3CreateForeignKey(pParse,0,&T,TA,R);}
@@ -349,7 +350,8 @@ tcons ::= CONSTRAINT nm(X).      {pParse->constraintName = X;}
 tcons ::= PRIMARY KEY LP sortlist(X) autoinc(I) RP onconf(R).
                                 {sqlite3AddPrimaryKey(pParse,X,R,I,0);}
 tcons ::= UNIQUE LP sortlist(X) RP onconf(R).
-                                 {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0);}
+                                 {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0,
+                                       SQLITE_IDXTYPE_UNIQUE);}
 tcons ::= CHECK LP expr(E) RP onconf.
                                 {sqlite3AddCheckConstraint(pParse,E.pExpr);}
 tcons ::= FOREIGN KEY LP eidlist(FA) RP
@@ -1198,7 +1200,7 @@ cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D)
        ON nm(Y) LP sortlist(Z) RP where_opt(W). {
  sqlite3CreateIndex(pParse, &X, &D, 
                     sqlite3SrcListAppend(pParse->db,0,&Y,0), Z, U,
-                      &S, W, SQLITE_SO_ASC, NE);
+                      &S, W, SQLITE_SO_ASC, NE, SQLITE_IDXTYPE_APPDEF);
 }

 %type uniqueflag {int}
@@ -1052,7 +1052,7 @@ void sqlite3Pragma(
  */
  case PragTyp_TABLE_INFO: if( zRight ){
    Table *pTab;
-    pTab = sqlite3FindTable(db, zRight, zDb);
+    pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb);
    if( pTab ){
      static const char *azCol[] = {
         "cid", "name", "type", "notnull", "dflt_value", "pk"
@@ -1028,7 +1028,7 @@ static void selectInnerLoop(
 */
 KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){
  int nExtra = (N+X)*(sizeof(CollSeq*)+1);
-  KeyInfo *p = sqlite3Malloc(sizeof(KeyInfo) + nExtra);
+  KeyInfo *p = sqlite3DbMallocRaw(db, sizeof(KeyInfo) + nExtra);
  if( p ){
    p->aSortOrder = (u8*)&p->aColl[N+X];
    p->nField = (u16)N;
@@ -1050,7 +1050,7 @@ void sqlite3KeyInfoUnref(KeyInfo *p){
  if( p ){
    assert( p->nRef>0 );
    p->nRef--;
-    if( p->nRef==0 ) sqlite3DbFree(0, p);
+    if( p->nRef==0 ) sqlite3DbFree(p->db, p);
  }
 }

@@ -3755,7 +3755,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    }
    zFile = azArg[1];
    zProc = nArg>=3 ? azArg[2] : 0;
-    /*    open_db(p, 0); */
+    open_db(p, 0);
    rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg);
    if( rc!=SQLITE_OK ){
      utf8_printf(stderr, "Error: %s\n", zErrMsg);
@@ -506,6 +506,7 @@ int sqlite3_exec(
 #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8))
 #define SQLITE_WARNING_AUTOINDEX       (SQLITE_WARNING | (1<<8))
 #define SQLITE_AUTH_USER               (SQLITE_AUTH | (1<<8))
+#define SQLITE_OK_LOAD_PERMANENTLY     (SQLITE_OK | (1<<8))

 /*
 ** CAPI3REF: Flags For File Open Operations
@@ -2485,7 +2485,7 @@ struct SrcList {
    int regReturn;    /* Register holding return address of addrFillSub */
    int regResult;    /* Registers holding results of a co-routine */
    struct {
-      u8 jointype;      /* Type of join between this able and the previous */
+      u8 jointype;      /* Type of join between this table and the previous */
      unsigned notIndexed :1;    /* True if there is a NOT INDEXED clause */
      unsigned isIndexedBy :1;   /* True if there is an INDEXED BY clause */
      unsigned isTabFunc :1;     /* True if table-valued-function syntax */
@@ -3617,8 +3617,8 @@ void sqlite3SrcListAssignCursors(Parse*, SrcList*);
 void sqlite3IdListDelete(sqlite3*, IdList*);
 void sqlite3SrcListDelete(sqlite3*, SrcList*);
 Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**);
-Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
-                          Expr*, int, int);
+void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
+                          Expr*, int, int, u8);
 void sqlite3DropIndex(Parse*, SrcList*, int);
 int sqlite3Select(Parse*, Select*, SelectDest*);
 Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*,
@@ -3671,8 +3671,10 @@ void sqlite3ExprIfTrue(Parse*, Expr*, int, int);
 void sqlite3ExprIfFalse(Parse*, Expr*, int, int);
 void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int);
 Table *sqlite3FindTable(sqlite3*,const char*, const char*);
-Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*);
-Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *);
+#define LOCATE_VIEW    0x01
+#define LOCATE_NOERR   0x02
+Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
+Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
 Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
 void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
 void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);
@@ -2325,6 +2325,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
  */
  case DB_EXISTS: 
  case DB_ONECOLUMN: {
+    Tcl_Obj *pResult = 0;
    DbEvalContext sEval;
    if( objc!=3 ){
      Tcl_WrongNumArgs(interp, 2, objv, "SQL");
@@ -2335,14 +2336,15 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
    rc = dbEvalStep(&sEval);
    if( choice==DB_ONECOLUMN ){
      if( rc==TCL_OK ){
-        Tcl_SetObjResult(interp, dbEvalColumnValue(&sEval, 0));
+        pResult = dbEvalColumnValue(&sEval, 0);
      }else if( rc==TCL_BREAK ){
        Tcl_ResetResult(interp);
      }
    }else if( rc==TCL_BREAK || rc==TCL_OK ){
-      Tcl_SetObjResult(interp, Tcl_NewBooleanObj(rc==TCL_OK));
+      pResult = Tcl_NewBooleanObj(rc==TCL_OK);
    }
    dbEvalFinalize(&sEval);
+    if( pResult ) Tcl_SetObjResult(interp, pResult);

    if( rc==TCL_BREAK ){
      rc = TCL_OK;
@@ -6584,6 +6584,7 @@ static int tclLoadStaticExtensionCmd(
 ){
  extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*);
  extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_csv_init(sqlite3*,char**,const sqlite3_api_routines*);
  extern int sqlite3_eval_init(sqlite3*,char**,const sqlite3_api_routines*);
  extern int sqlite3_fileio_init(sqlite3*,char**,const sqlite3_api_routines*);
  extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*);
@@ -6601,6 +6602,7 @@ static int tclLoadStaticExtensionCmd(
  } aExtension[] = {
    { "amatch",                sqlite3_amatch_init               },
    { "closure",               sqlite3_closure_init              },
+    { "csv",                   sqlite3_csv_init                  },
    { "eval",                  sqlite3_eval_init                 },
    { "fileio",                sqlite3_fileio_init               },
    { "fuzzer",                sqlite3_fuzzer_init               },
@@ -385,8 +385,7 @@ static int btree_payload_size(
  const char **argv      /* Text of each argument */
 ){
  BtCursor *pCur;
-  int n2;
-  u64 n1;
+  u32 n;
  char zBuf[50];

  if( argc!=2 ){
@@ -396,17 +395,9 @@ static int btree_payload_size(
  }
  pCur = sqlite3TestTextToPtr(argv[1]);
  sqlite3BtreeEnter(pCur->pBtree);
-
-  /* The cursor may be in "require-seek" state. If this is the case, the
-  ** call to BtreeDataSize() will fix it. */
-  sqlite3BtreeDataSize(pCur, (u32*)&n2);
-  if( pCur->apPage[pCur->iPage]->intKey ){
-    n1 = 0;
-  }else{
-    sqlite3BtreeKeySize(pCur, (i64*)&n1);
-  }
+  n = sqlite3BtreePayloadSize(pCur);
  sqlite3BtreeLeave(pCur->pBtree);
-  sqlite3_snprintf(sizeof(zBuf),zBuf, "%d", (int)(n1+n2));
+  sqlite3_snprintf(sizeof(zBuf),zBuf, "%u", n);
  Tcl_AppendResult(interp, zBuf, 0);
  return SQLITE_OK;
 }
@@ -2016,6 +2016,7 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
      if( (flags1 | flags3)&MEM_Str ){
        if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
          applyNumericAffinity(pIn1,0);
+          flags3 = pIn3->flags;
        }
        if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
          applyNumericAffinity(pIn3,0);
@@ -2028,6 +2029,7 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
        sqlite3VdbeMemStringify(pIn1, encoding, 1);
        testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) );
        flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask);
+        flags3 = pIn3->flags;
      }
      if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){
        testcase( pIn3->flags & MEM_Int );
@@ -2380,7 +2382,6 @@ case OP_NotNull: {            /* same as TK_NOTNULL, jump, in1 */
 ** skipped for length() and all content loading can be skipped for typeof().
 */
 case OP_Column: {
-  i64 payloadSize64; /* Number of bytes in the record */
  int p2;            /* column number to retrieve */
  VdbeCursor *pC;    /* The VDBE cursor */
  BtCursor *pCrsr;   /* The BTree cursor */
@@ -2433,22 +2434,9 @@ case OP_Column: {
    }else{
      assert( pC->eCurType==CURTYPE_BTREE );
      assert( pCrsr );
-      if( pC->isTable==0 ){
-        assert( sqlite3BtreeCursorIsValid(pCrsr) );
-        VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64);
-        assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */
-        /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the
-        ** payload size, so it is impossible for payloadSize64 to be
-        ** larger than 32 bits. */
-        assert( (payloadSize64 & SQLITE_MAX_U32)==(u64)payloadSize64 );
-        pC->aRow = sqlite3BtreeKeyFetch(pCrsr, &avail);
-        pC->payloadSize = (u32)payloadSize64;
-      }else{
-        assert( sqlite3BtreeCursorIsValid(pCrsr) );
-        VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &pC->payloadSize);
-        assert( rc==SQLITE_OK );   /* DataSize() cannot fail */
-        pC->aRow = sqlite3BtreeDataFetch(pCrsr, &avail);
-      }
+      assert( sqlite3BtreeCursorIsValid(pCrsr) );
+      pC->payloadSize = sqlite3BtreePayloadSize(pCrsr);
+      pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &avail);
      assert( avail<=65536 );  /* Maximum page size is 64KiB */
      if( pC->payloadSize <= (u32)avail ){
        pC->szRow = pC->payloadSize;
@@ -4201,8 +4189,7 @@ case OP_NewRowid: {           /* out2 */
        v = 1;   /* IMP: R-61914-48074 */
      }else{
        assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) );
-        rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
-        assert( rc==SQLITE_OK );   /* Cannot fail following BtreeLast() */
+        v = sqlite3BtreeIntegerKey(pC->uc.pCursor);
        if( v>=MAX_ROWID ){
          pC->useRandomRowid = 1;
        }else{
@@ -4459,8 +4446,7 @@ case OP_Delete: {
    /* If p5 is zero, the seek operation that positioned the cursor prior to
    ** OP_Delete will have also set the pC->movetoTarget field to the rowid of
    ** the row that is being deleted */
-    i64 iKey = 0;
-    sqlite3BtreeKeySize(pC->uc.pCursor, &iKey);
+    i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor);
    assert( pC->movetoTarget==iKey );
  }
 #endif
@@ -4476,7 +4462,7 @@ case OP_Delete: {
    zDb = db->aDb[pC->iDb].zName;
    pTab = pOp->p4.pTab;
    if( (pOp->p5 & OPFLAG_SAVEPOSITION)!=0 && pC->isTable ){
-      sqlite3BtreeKeySize(pC->uc.pCursor, &pC->movetoTarget);
+      pC->movetoTarget = sqlite3BtreeIntegerKey(pC->uc.pCursor);
    }
  }else{
    zDb = 0;   /* Not needed.  Silence a compiler warning. */
@@ -4630,7 +4616,6 @@ case OP_RowData: {
  VdbeCursor *pC;
  BtCursor *pCrsr;
  u32 n;
-  i64 n64;

  pOut = &aMem[pOp->p2];
  memAboutToChange(p, pOut);
@@ -4662,20 +4647,9 @@ case OP_RowData: {
  if( rc!=SQLITE_OK ) goto abort_due_to_error;
 #endif

-  if( pC->isTable==0 ){
-    assert( !pC->isTable );
-    VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &n64);
-    assert( rc==SQLITE_OK );    /* True because of CursorMoveto() call above */
-    if( n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){
-      goto too_big;
-    }
-    n = (u32)n64;
-  }else{
-    VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &n);
-    assert( rc==SQLITE_OK );    /* DataSize() cannot fail */
-    if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
-      goto too_big;
-    }
+  n = sqlite3BtreePayloadSize(pCrsr);
+  if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
+    goto too_big;
  }
  testcase( n==0 );
  if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){
@@ -4740,8 +4714,7 @@ case OP_Rowid: {                 /* out2 */
      pOut->flags = MEM_Null;
      break;
    }
-    rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
-    assert( rc==SQLITE_OK );  /* Always so because of CursorRestore() above */
+    v = sqlite3BtreeIntegerKey(pC->uc.pCursor);
  }
  pOut->u.i = v;
  break;
@@ -1650,8 +1650,7 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
    u32 nRec;
    u8 *aRec;

-    rc = sqlite3BtreeDataSize(p->pCsr->uc.pCursor, &nRec);
-    if( rc!=SQLITE_OK ) goto preupdate_old_out;
+    nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor);
    aRec = sqlite3DbMallocRaw(db, nRec);
    if( !aRec ) goto preupdate_old_out;
    rc = sqlite3BtreeData(p->pCsr->uc.pCursor, 0, nRec, aRec);
@@ -4315,8 +4315,7 @@ int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){
  ** this code can safely assume that nCellKey is 32-bits  
  */
  assert( sqlite3BtreeCursorIsValid(pCur) );
-  VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey);
-  assert( rc==SQLITE_OK );     /* pCur is always valid so KeySize cannot fail */
+  nCellKey = sqlite3BtreePayloadSize(pCur);
  assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey );

  /* Read in the complete content of the index entry */
@@ -4393,8 +4392,7 @@ int sqlite3VdbeIdxKeyCompare(
  assert( pC->eCurType==CURTYPE_BTREE );
  pCur = pC->uc.pCursor;
  assert( sqlite3BtreeCursorIsValid(pCur) );
-  VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey);
-  assert( rc==SQLITE_OK );    /* pCur is always valid so KeySize cannot fail */
+  nCellKey = sqlite3BtreePayloadSize(pCur);
  /* nCellKey will always be between 0 and 0xffffffff because of the way
  ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */
  if( nCellKey<=0 || nCellKey>0x7fffffff ){
@@ -415,7 +415,7 @@ static int blobReadWrite(
      ** anyhow.
      */
      sqlite3_int64 iKey;
-      sqlite3BtreeKeySize(p->pCsr, &iKey);
+      iKey = sqlite3BtreeIntegerKey(p->pCsr);
      sqlite3VdbePreUpdateHook(
          v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1
      );
@@ -991,11 +991,7 @@ int sqlite3VdbeMemFromBtree(
  /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() 
  ** that both the BtShared and database handle mutexes are held. */
  assert( (pMem->flags & MEM_RowSet)==0 );
-  if( key ){
-    zData = (char *)sqlite3BtreeKeyFetch(pCur, &available);
-  }else{
-    zData = (char *)sqlite3BtreeDataFetch(pCur, &available);
-  }
+  zData = (char *)sqlite3BtreePayloadFetch(pCur, &available);
  assert( zData!=0 );

  if( offset+amt<=available ){
@@ -754,10 +754,24 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
     && (pParse->pNewTable->tabFlags & TF_Virtual)==0
    ){
      if( !pTab->aCol ){
-        pTab->aCol = pParse->pNewTable->aCol;
-        pTab->nCol = pParse->pNewTable->nCol;
-        pParse->pNewTable->nCol = 0;
-        pParse->pNewTable->aCol = 0;
+        Table *pNew = pParse->pNewTable;
+        Index *pIdx;
+        pTab->aCol = pNew->aCol;
+        pTab->nCol = pNew->nCol;
+        pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid);
+        pNew->nCol = 0;
+        pNew->aCol = 0;
+        assert( pTab->pIndex==0 );
+        if( !HasRowid(pNew) && pCtx->pVTable->pMod->pModule->xUpdate!=0 ){
+          rc = SQLITE_ERROR;
+        }
+        pIdx = pNew->pIndex;
+        if( pIdx ){
+          assert( pIdx->pNext==0 );
+          pTab->pIndex = pIdx;
+          pNew->pIndex = 0;
+          pIdx->pTable = pTab;
+        }
      }
      pCtx->bDeclared = 1;
    }else{
@@ -1087,7 +1101,7 @@ void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){
 }

 /*
-** Check to see if virtual tale module pMod can be have an eponymous
+** Check to see if virtual table module pMod can be have an eponymous
 ** virtual table instance.  If it can, create one if one does not already
 ** exist. Return non-zero if the eponymous virtual table instance exists
 ** when this routine returns, and return zero if it does not exist.
@@ -1104,17 +1118,18 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
  const sqlite3_module *pModule = pMod->pModule;
  Table *pTab;
  char *zErr = 0;
-  int nName;
  int rc;
  sqlite3 *db = pParse->db;
  if( pMod->pEpoTab ) return 1;
  if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0;
-  nName = sqlite3Strlen30(pMod->zName) + 1;
-  pTab = sqlite3DbMallocZero(db, sizeof(Table) + nName);
+  pTab = sqlite3DbMallocZero(db, sizeof(Table));
  if( pTab==0 ) return 0;
+  pTab->zName = sqlite3DbStrDup(db, pMod->zName);
+  if( pTab->zName==0 ){
+    sqlite3DbFree(db, pTab);
+    return 0;
+  }
  pMod->pEpoTab = pTab;
-  pTab->zName = (char*)&pTab[1];
-  memcpy(pTab->zName, pMod->zName, nName);
  pTab->nRef = 1;
  pTab->pSchema = db->aDb[0].pSchema;
  pTab->tabFlags |= TF_Virtual;
@@ -1140,9 +1155,11 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
 void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){
  Table *pTab = pMod->pEpoTab;
  if( pTab!=0 ){
-    sqlite3DeleteColumnNames(db, pTab);
-    sqlite3VtabClear(db, pTab);
-    sqlite3DbFree(db, pTab);
+    /* Mark the table as Ephemeral prior to deleting it, so that the
+    ** sqlite3DeleteTable() routine will know that it is not stored in 
+    ** the schema. */
+    pTab->tabFlags |= TF_Ephemeral;
+    sqlite3DeleteTable(db, pTab);
    pMod->pEpoTab = 0;
  }
 }
@@ -2701,6 +2701,7 @@ static int whereLoopAddBtree(
          pNew->rSetup += 24;
        }
        ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
+        if( pNew->rSetup<0 ) pNew->rSetup = 0;
        /* TUNING: Each index lookup yields 20 rows in the table.  This
        ** is more than the usual guess of 10 rows, since we have no way
        ** of knowing how selective the index will ultimately be.  It would
@@ -2761,6 +2762,7 @@ static int whereLoopAddBtree(
      /* Full scan via index */
      if( b
       || !HasRowid(pTab)
+       || pProbe->pPartIdxWhere!=0
       || ( m==0
         && pProbe->bUnordered==0
         && (pProbe->szIdxRow<pTab->szTabRow)
@@ -3239,6 +3241,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
      mPrereq = mPrior;
    }
    priorJointype = pItem->fg.jointype;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
    if( IsVirtual(pItem->pTab) ){
      struct SrcList_item *p;
      for(p=&pItem[1]; p<pEnd; p++){
@@ -3247,7 +3250,9 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
        }
      }
      rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
-    }else{
+    }else
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+    {
      rc = whereLoopAddBtree(pBuilder, mPrereq);
    }
    if( rc==SQLITE_OK ){
@@ -626,6 +626,38 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){
  return WRC_Continue;
 }

+/*
+** Test whether or not expression pExpr, which was part of a WHERE clause,
+** should be included in the cursor-hint for a table that is on the rhs
+** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the 
+** expression is not suitable.
+**
+** An expression is unsuitable if it might evaluate to non NULL even if
+** a TK_COLUMN node that does affect the value of the expression is set
+** to NULL. For example:
+**
+**   col IS NULL
+**   col IS NOT NULL
+**   coalesce(col, 1)
+**   CASE WHEN col THEN 0 ELSE 1 END
+*/
+static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){
+  if( pExpr->op==TK_IS 
+   || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT 
+   || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE 
+  ){
+    pWalker->eCode = 1;
+  }else if( pExpr->op==TK_FUNCTION ){
+    int d1;
+    char d2[3];
+    if( 0==sqlite3IsLikeFunction(pWalker->pParse->db, pExpr, &d1, d2) ){
+      pWalker->eCode = 1;
+    }
+  }
+
+  return WRC_Continue;
+}
+

 /*
 ** This function is called on every node of an expression tree used as an
@@ -678,6 +710,7 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
 ** Insert an OP_CursorHint instruction if it is appropriate to do so.
 */
 static void codeCursorHint(
+  struct SrcList_item *pTabItem,  /* FROM clause item */
  WhereInfo *pWInfo,    /* The where clause */
  WhereLevel *pLevel,   /* Which loop to provide hints for */
  WhereTerm *pEndRange  /* Hint this end-of-scan boundary term if not NULL */
@@ -708,7 +741,42 @@ static void codeCursorHint(
    pTerm = &pWC->a[i];
    if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
    if( pTerm->prereqAll & pLevel->notReady ) continue;
-    if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
+
+    /* Any terms specified as part of the ON(...) clause for any LEFT 
+    ** JOIN for which the current table is not the rhs are omitted
+    ** from the cursor-hint. 
+    **
+    ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms 
+    ** that were specified as part of the WHERE clause must be excluded.
+    ** This is to address the following:
+    **
+    **   SELECT ... t1 LEFT JOIN t2 ON (t1.a=t2.b) WHERE t2.c IS NULL;
+    **
+    ** Say there is a single row in t2 that matches (t1.a=t2.b), but its
+    ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is 
+    ** pushed down to the cursor, this row is filtered out, causing
+    ** SQLite to synthesize a row of NULL values. Which does match the
+    ** WHERE clause, and so the query returns a row. Which is incorrect.
+    **
+    ** For the same reason, WHERE terms such as:
+    **
+    **   WHERE 1 = (t2.c IS NULL)
+    **
+    ** are also excluded. See codeCursorHintIsOrFunction() for details.
+    */
+    if( pTabItem->fg.jointype & JT_LEFT ){
+      Expr *pExpr = pTerm->pExpr;
+      if( !ExprHasProperty(pExpr, EP_FromJoin) 
+       || pExpr->iRightJoinTable!=pTabItem->iCursor
+      ){
+        sWalker.eCode = 0;
+        sWalker.xExprCallback = codeCursorHintIsOrFunction;
+        sqlite3WalkExpr(&sWalker, pTerm->pExpr);
+        if( sWalker.eCode ) continue;
+      }
+    }else{
+      if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
+    }

    /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize
    ** the cursor.  These terms are not needed as hints for a pure range
@@ -742,7 +810,7 @@ static void codeCursorHint(
  }
 }
 #else
-# define codeCursorHint(A,B,C)  /* No-op */
+# define codeCursorHint(A,B,C,D)  /* No-op */
 #endif /* SQLITE_ENABLE_CURSOR_HINTS */

 /*
@@ -998,7 +1066,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
      pStart = pEnd;
      pEnd = pTerm;
    }
-    codeCursorHint(pWInfo, pLevel, pEnd);
+    codeCursorHint(pTabItem, pWInfo, pLevel, pEnd);
    if( pStart ){
      Expr *pX;             /* The expression that defines the start bound */
      int r1, rTemp;        /* Registers for holding the start boundary */
@@ -1212,7 +1280,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
    ** and store the values of those terms in an array of registers
    ** starting at regBase.
    */
-    codeCursorHint(pWInfo, pLevel, pRangeEnd);
+    codeCursorHint(pTabItem, pWInfo, pLevel, pRangeEnd);
    regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff);
    assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq );
    if( zStartAff ) cEndAff = zStartAff[nEq];
@@ -1660,7 +1728,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
      ** a pseudo-cursor.  No need to Rewind or Next such cursors. */
      pLevel->op = OP_Noop;
    }else{
-      codeCursorHint(pWInfo, pLevel, 0);
+      codeCursorHint(pTabItem, pWInfo, pLevel, 0);
      pLevel->op = aStep[bRev];
      pLevel->p1 = iCur;
      pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);
@@ -568,7 +568,9 @@ static void exprAnalyzeOrTerm(
        if( !db->mallocFailed ){
          for(j=0, pAndTerm=pAndWC->a; j<pAndWC->nTerm; j++, pAndTerm++){
            assert( pAndTerm->pExpr );
-            if( allowedOp(pAndTerm->pExpr->op) ){
+            if( allowedOp(pAndTerm->pExpr->op) 
+             || pAndTerm->eOperator==WO_MATCH 
+            ){
              b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor);
            }
          }
@@ -1122,7 +1124,7 @@ static void exprAnalyze(
  ** virtual tables.  The native query optimizer does not attempt
  ** to do anything with MATCH functions.
  */
-  if( isMatchOfColumn(pExpr, &eOp2) ){
+  if( pWC->op==TK_AND && isMatchOfColumn(pExpr, &eOp2) ){
    int idxNew;
    Expr *pRight, *pLeft;
    WhereTerm *pNewTerm;
@@ -0,0 +1,170 @@
+# 2016 May 29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindex3
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Virtual table callback for a virtual table named $tbl.
+#
+# The table created is:
+#
+#      "CREATE TABLE t1 (a, b, c)"
+#
+# This virtual table supports both LIKE and = operators on all columns.
+#  
+proc vtab_cmd {bOmit method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c)"
+    }
+
+    xBestIndex {
+      foreach {clist orderby mask} $args {}
+
+      set ret [list]
+      set use use
+      if {$bOmit} {set use omit}
+
+      for {set i 0} {$i < [llength $clist]} {incr i} {
+        array unset C
+        array set C [lindex $clist $i]
+        if {$C(usable) && ($C(op)=="like" || $C(op)=="eq")} {
+          lappend ret $use $i
+          lappend ret idxstr 
+          lappend ret "[lindex {a b c} $C(column)] [string toupper $C(op)] ?"
+          break
+        }
+      }
+
+      if {$ret==""} {
+        lappend ret cost 1000000 rows 1000000
+      } else {
+        lappend ret cost 100 rows 10
+      }
+      return $ret
+    }
+
+    xFilter {
+      foreach {idxnum idxstr param} $args {}
+      set where ""
+      if {$bOmit && $idxstr != ""} {
+        set where " WHERE [string map [list ? '$param' EQ =] $idxstr]"
+      }
+      return [list sql "SELECT rowid, * FROM ttt$where"]
+    }
+  }
+  return ""
+}
+
+register_tcl_module db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING tcl("vtab_cmd 0");
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM t1 WHERE a LIKE 'abc';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
+}
+
+do_eqp_test 1.2 {
+  SELECT * FROM t1 WHERE a = 'abc';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
+}
+
+do_eqp_test 1.3 {
+  SELECT * FROM t1 WHERE a = 'abc' OR b = 'def';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
+}
+
+do_eqp_test 1.4 {
+  SELECT * FROM t1 WHERE a LIKE 'abc%' OR b = 'def';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
+}
+
+do_execsql_test 1.5 {
+  CREATE TABLE ttt(a, b, c);
+
+  INSERT INTO ttt VALUES(1, 'two',   'three');
+  INSERT INTO ttt VALUES(2, 'one',   'two');
+  INSERT INTO ttt VALUES(3, 'three', 'one');
+  INSERT INTO ttt VALUES(4, 'y',     'one');
+  INSERT INTO ttt VALUES(5, 'x',     'two');
+  INSERT INTO ttt VALUES(6, 'y',     'three');
+}
+
+foreach omit {0 1} {
+  do_execsql_test 1.6.$omit.0 "
+    DROP TABLE t1;
+    CREATE VIRTUAL TABLE t1 USING tcl('vtab_cmd $omit');
+  "
+  do_execsql_test 1.6.$omit.1 { 
+    SELECT rowid FROM t1 WHERE c LIKE 'o%'
+  } {3 4}
+
+  do_execsql_test 1.6.$omit.2 { 
+    SELECT rowid FROM t1 WHERE c LIKE 'o%' OR b='y'
+  } {3 4 6}
+
+  do_execsql_test 1.6.$omit.3 { 
+    SELECT rowid FROM t1 WHERE c = 'three' OR c LIKE 'o%'
+  } {1 6 3 4}
+}
+
+#-------------------------------------------------------------------------
+# Test the same pattern works with ordinary tables.
+#
+do_execsql_test 2.1 {
+  CREATE TABLE t2(x TEXT COLLATE nocase, y TEXT);
+  CREATE INDEX t2x ON t2(x COLLATE nocase);
+  CREATE INDEX t2y ON t2(y);
+}
+
+do_eqp_test 2.2 {
+  SELECT * FROM t2 WHERE x LIKE 'abc%' OR y = 'def'
+} {
+  0 0 0 {SEARCH TABLE t2 USING INDEX t2x (x>? AND x<?)}
+  0 0 0 {SEARCH TABLE t2 USING INDEX t2y (y=?)}
+}
+
+#-------------------------------------------------------------------------
+# Test that any PRIMARY KEY within a sqlite3_decl_vtab() CREATE TABLE 
+# statement is currently ignored.
+#
+proc vvv_command {method args} {
+  switch -- $method {
+    xConnect { return "CREATE TABLE t1(a PRIMARY KEY, b, c)" }
+  }
+}
+proc yyy_command {method args} {
+  switch -- $method {
+    xConnect { return "CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b))" }
+  }
+}
+
+do_execsql_test 3.1 { CREATE VIRTUAL TABLE t3 USING tcl('vvv_command') }
+do_execsql_test 3.2 { CREATE VIRTUAL TABLE t4 USING tcl('yyy_command') }
+
+finish_test
+
@@ -0,0 +1,112 @@
+# 2016-06-02
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# Test cases for CSV virtual table.
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix closure01
+
+ifcapable !vtab||!cte { finish_test ; return }
+
+load_static_extension db csv
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE temp.t1 USING csv(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4
+  );
+  SELECT * FROM t1 WHERE c1=10;
+} {9 10 11 12}
+do_execsql_test 1.1 {
+  SELECT * FROM t1 WHERE c1='10';
+} {9 10 11 12}
+do_execsql_test 1.2 {
+  SELECT rowid FROM t1;
+} {1 2 3 4}
+
+do_execsql_test 2.0 {
+  DROP TABLE t1;
+  CREATE VIRTUAL TABLE temp.t2 USING csv(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4,
+    schema='CREATE TABLE t2(a INT, b TEXT, c REAL, d BLOB)'
+  );
+  SELECT * FROM t2 WHERE a=9;
+} {9 10 11 12}
+do_execsql_test 2.1 {
+  SELECT * FROM t2 WHERE b=10;
+} {9 10 11 12}
+do_execsql_test 2.2 {
+  SELECT * FROM t2 WHERE c=11;
+} {9 10 11 12}
+do_execsql_test 2.3 {
+  SELECT * FROM t2 WHERE d=12;
+} {}
+do_execsql_test 2.4 {
+  SELECT * FROM t2 WHERE d='12';
+} {9 10 11 12}
+do_execsql_test 2.5 {
+  SELECT * FROM t2 WHERE a='9';
+} {9 10 11 12}
+
+do_execsql_test 3.0 {
+  DROP TABLE t2;
+  CREATE VIRTUAL TABLE temp.t3 USING csv(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4,
+    schema=
+      'CREATE TABLE t3(a PRIMARY KEY,b TEXT,c TEXT,d TEXT) WITHOUT ROWID',
+    testflags=1
+  );
+  SELECT a FROM t3 WHERE b=6 OR c=7 OR d=12 ORDER BY +a;
+} {5 9}
+do_execsql_test 3.1 {
+  SELECT a FROM t3 WHERE +b=6 OR c=7 OR d=12 ORDER BY +a;
+} {5 9}
+
+# The rowid column is not visible on a WITHOUT ROWID virtual table
+do_catchsql_test 3.2 {
+  SELECT rowid, a FROM t3;
+} {1 {no such column: rowid}}
+
+do_catchsql_test 4.0 {
+  DROP TABLE t3;
+  CREATE VIRTUAL TABLE temp.t4 USING csv_wr(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4,
+    schema=
+      'CREATE TABLE t3(a PRIMARY KEY,b TEXT,c TEXT,d TEXT) WITHOUT ROWID',
+    testflags=1
+  );
+} {1 {vtable constructor failed: t4}}
+
+finish_test
@@ -196,9 +196,10 @@ set tc 1
 foreach opt $opts {
  do_test ctime-2.5.$tc {
    set N [ expr {$tc-1} ]
-    set ans1 [ catchsql {
+    set ans1 [catch {db one {
      SELECT sqlite_compileoption_get($N);
-    } ]
+    }} msg]
+    lappend ans1 $msg
    set ans2 [ catchsql {
      SELECT sqlite_compileoption_used($opt);
    } ]
@@ -0,0 +1,180 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library. The
+# focus is on testing that cursor-hints are correct for queries
+# involving LEFT JOIN.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix cursorhint2
+
+ifcapable !cursorhints {
+  finish_test
+  return
+}
+
+proc extract_hints {sql} {
+
+  db eval "SELECT tbl_name, rootpage FROM sqlite_master where rootpage" {
+    set lookup($rootpage) $tbl_name
+  }
+
+  set ret [list]
+  db eval "EXPLAIN $sql" a {
+    switch -- $a(opcode) {
+      OpenRead {
+        set csr($a(p1)) $lookup($a(p2))
+      }
+      CursorHint { 
+        lappend ret $csr($a(p1)) $a(p4) 
+      }
+    }
+  }
+
+  set ret
+}
+
+proc do_extract_hints_test {tn sql ret} {
+  uplevel [list do_test $tn [list extract_hints $sql] [list {*}$ret]]
+}
+
+do_execsql_test 1.0 {
+  PRAGMA automatic_index = 0;
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+  CREATE TABLE t3(e, f);
+}
+
+do_extract_hints_test 1.1 {
+  SELECT * FROM t1 WHERE a=1;
+} {
+  t1 EQ(c0,1)
+}
+
+do_extract_hints_test 1.2 {
+  SELECT * FROM t1 CROSS JOIN t2 ON (a=c) WHERE d IS NULL;
+} {
+  t2 {AND(ISNULL(c1),EQ(r[1],c0))}
+}
+
+do_extract_hints_test 1.3 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c) WHERE d IS NULL;
+} {
+  t2 {EQ(r[2],c0)}
+}
+
+do_extract_hints_test 1.4 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND a=10) WHERE d IS NULL;
+} {
+  t2 {AND(EQ(r[2],c0),EQ(r[3],10))}
+}
+
+do_extract_hints_test 1.5 {
+  SELECT * FROM t1 CROSS JOIN t2 ON (a=c AND a=10) WHERE d IS NULL;
+} {
+  t1 EQ(c0,10) t2 {AND(ISNULL(c1),EQ(r[3],c0))}
+}
+
+do_extract_hints_test 1.6 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c) LEFT JOIN t3 ON (d=f);
+} {
+  t2 {EQ(r[2],c0)} t3 {EQ(r[6],c1)}
+}
+
+do_extract_hints_test 1.7 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND d=e) LEFT JOIN t3 ON (d=f);
+} {
+  t2 {EQ(r[2],c0)} t3 {AND(EQ(r[6],c0),EQ(r[7],c1))}
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+  CREATE TABLE x1(x, y);
+  CREATE TABLE x2(a, b);
+}
+
+do_extract_hints_test 2.1 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NULL;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.2 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS +NULL;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.3 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = (b IS NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.4 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = coalesce(b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.5 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = coalesce(b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.6 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.7 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT +NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.8 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NOT +NULL
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.9 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE CASE b WHEN 0 THEN 0 ELSE 1 END;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.10 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b = 32+32
+} {
+  x2 {AND(EQ(c1,ADD(32,32)),EQ(c0,r[2]))}
+}
+
+do_extract_hints_test 2.11 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b LIKE 'abc%'
+} {
+  x2 {AND(expr,EQ(c0,r[2]))}
+}
+
+do_extract_hints_test 2.11 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE coalesce(x2.b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+finish_test
+
@@ -376,4 +376,15 @@ do_execsql_test index6-10.3eqp {
  SELECT e FROM t10 WHERE a=1 AND b=2 ORDER BY d DESC;
 } {~/USING INDEX t10x/}

+# A partial index will be used for a full table scan, where possible
+do_execsql_test index6-11.1 {
+  CREATE TABLE t11(a,b,c);
+  CREATE INDEX t11x ON t11(a) WHERE b<>99;
+  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
+} {/USING INDEX t11x/}
+do_execsql_test index6-11.2 {
+  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
+} {/USING INDEX t11x/}
+  
+
 finish_test
@@ -1434,5 +1434,14 @@ do_catchsql_test 5.4 {
  SELECT * FROM t8 UNION SELECT * FROM t9 ORDER BY a+b COLLATE NOCASE
 } {1 {1st ORDER BY term does not match any column in the result set}}

+do_execsql_test 6.1 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(a INTEGER);
+  CREATE TABLE t2(b TEXT);
+  INSERT INTO t2(b) VALUES('12345');
+  SELECT * FROM (SELECT a FROM t1 UNION SELECT b FROM t2) WHERE a=a;
+} {12345}
+

 finish_test
@@ -1216,6 +1216,10 @@ static void displayLinuxIoStats(FILE *out){
 }   
 #endif

+#if SQLITE_VERSION_NUMBER<3006018
+#  define sqlite3_sourceid(X) "(before 3.6.18)"
+#endif
+
 int main(int argc, char **argv){
  int doAutovac = 0;            /* True for --autovacuum */
  int cacheSize = 0;            /* Desired cache size.  0 means default */
@@ -32,6 +32,9 @@ db func a_string a_string

 register_dbstat_vtab db
 do_execsql_test stat-0.0 {
+  PRAGMA table_info(dbstat);
+} {/0 name TEXT .* 1 path TEXT .* 9 pgsize INTEGER/}
+do_execsql_test stat-0.1 {
  PRAGMA auto_vacuum = OFF;
  CREATE VIRTUAL TABLE temp.stat USING dbstat;
  SELECT * FROM stat;
@@ -635,6 +635,39 @@ do_test tcl-14.2 {
  db one {SELECT x FROM t6 WHERE xCall()!='value'}
 } {}

+# Verify that the "exists" and "onecolumn" methods work when
+# a "profile" is registered.
+#
+catch {db close}
+sqlite3 db :memory:
+proc noop-profile {args} {
+  return
+}
+do_test tcl-15.0 {
+  db eval {CREATE TABLE t1(a); INSERT INTO t1 VALUES(1),(2),(3);}
+  db onecolumn {SELECT a FROM t1 WHERE a>2}
+} {3}
+do_test tcl-15.1 {
+  db exists {SELECT a FROM t1 WHERE a>2}
+} {1}
+do_test tcl-15.2 {
+  db exists {SELECT a FROM t1 WHERE a>3}
+} {0}
+db profile noop-profile
+do_test tcl-15.3 {
+  db onecolumn {SELECT a FROM t1 WHERE a>2}
+} {3}
+do_test tcl-15.4 {
+  db exists {SELECT a FROM t1 WHERE a>2}
+} {1}
+do_test tcl-15.5 {
+  db exists {SELECT a FROM t1 WHERE a>3}
+} {0}
+
+
+
+
+


 finish_test
@@ -35,6 +35,14 @@ do_execsql_test 1.0 {

 faultsim_save_and_close

+# The error message is different on unix and windows
+#
+if {$::tcl_platform(platform)=="windows"} {
+ set msg "child killed: unknown signal"
+} else {
+ set msg "child process exited abnormally"
+}
+
 for {set nExtra 0} {$nExtra < 10} {incr nExtra} {
  for {set i 0} {$i < 10} {incr i} {
    do_test 1.nExtra=$nExtra.i=$i.1 {
@@ -63,7 +71,7 @@ for {set nExtra 0} {$nExtra < 10} {incr nExtra} {
    
      set r [catch { exec [info nameofexec] crash.tcl >@stdout } msg]
      list $r $msg
-    } {1 {child process exited abnormally}}
+    } "1 {$msg}"
  
    do_execsql_test 1.nExtra=$nExtra.i=$i.2 { 
      SELECT count(*) FROM t1;
@@ -45,7 +45,7 @@ do_test where3-1.1 {
  }
 } {222 two 2 222 {} {}}

-ifcapable explain {
+ifcapable explain&&!cursorhints {
  do_test where3-1.1.1 {
     explain_no_trace {SELECT * FROM t1, t2 LEFT JOIN t3 ON q=x
                        WHERE p=2 AND a=q}
@@ -86,7 +86,7 @@ do_test where3-1.2 {
  }
 } {1 {Value for C1.1} {Value for C2.1} 2 {} {Value for C2.2} 3 {Value for C1.3} {Value for C2.3}}

-ifcapable explain {
+ifcapable explain&&!cursorhints {
  do_test where3-1.2.1 {
     explain_no_trace {
       SELECT parent1.parent1key, child1.value, child2.value
@@ -0,0 +1,506 @@
+/*
+** 2016-06-07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This is a utility program that computes an SHA1 hash on the content
+** of an SQLite database.
+**
+** The hash is computed over just the content of the database.  Free
+** space inside of the database file, and alternative on-disk representations
+** of the same content (ex: UTF8 vs UTF16) do not affect the hash.  So,
+** for example, the database file page size, encoding, and auto_vacuum setting
+** can all be changed without changing the hash.
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <string.h>
+#include <assert.h>
+#include "sqlite3.h"
+
+/* Context for the SHA1 hash */
+typedef struct SHA1Context SHA1Context;
+struct SHA1Context {
+  unsigned int state[5];
+  unsigned int count[2];
+  unsigned char buffer[64];
+};
+
+/*
+** All global variables are gathered into the "g" singleton.
+*/
+struct GlobalVars {
+  const char *zArgv0;       /* Name of program */
+  unsigned fDebug;          /* Debug flags */
+  sqlite3 *db;              /* The database connection */
+  SHA1Context cx;           /* SHA1 hash context */
+} g;
+
+/*
+** Debugging flags
+*/
+#define DEBUG_FULLTRACE   0x00000001   /* Trace hash to stderr */
+
+/******************************************************************************
+** The Hash Engine
+**
+** Modify these routines (and appropriate state fields in global variable 'g')
+** in order to compute a different (better?) hash of the database.
+*/
+/*
+ * blk0() and blk() perform the initial expand.
+ * I got the idea of expanding during the round function from SSLeay
+ *
+ * blk0le() for little-endian and blk0be() for big-endian.
+ */
+#if __GNUC__ && (defined(__i386__) || defined(__x86_64__))
+/*
+ * GCC by itself only generates left rotates.  Use right rotates if
+ * possible to be kinder to dinky implementations with iterative rotate
+ * instructions.
+ */
+#define SHA_ROT(op, x, k) \
+        ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; })
+#define rol(x,k) SHA_ROT("roll", x, k)
+#define ror(x,k) SHA_ROT("rorl", x, k)
+
+#else
+/* Generic C equivalent */
+#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r))
+#define rol(x,k) SHA_ROT(x,k,32-(k))
+#define ror(x,k) SHA_ROT(x,32-(k),k)
+#endif
+
+
+#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \
+    |(rol(block[i],8)&0x00FF00FF))
+#define blk0be(i) block[i]
+#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \
+    ^block[(i+2)&15]^block[i&15],1))
+
+/*
+ * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1
+ *
+ * Rl0() for little-endian and Rb0() for big-endian.  Endianness is
+ * determined at run-time.
+ */
+#define Rl0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define Rb0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R1(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R2(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2);
+#define R3(v,w,x,y,z,i) \
+    z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2);
+#define R4(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2);
+
+/*
+ * Hash a single 512-bit block. This is the core of the algorithm.
+ */
+#define a qq[0]
+#define b qq[1]
+#define c qq[2]
+#define d qq[3]
+#define e qq[4]
+
+void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){
+  unsigned int qq[5]; /* a, b, c, d, e; */
+  static int one = 1;
+  unsigned int block[16];
+  memcpy(block, buffer, 64);
+  memcpy(qq,state,5*sizeof(unsigned int));
+
+  /* Copy g.cx.state[] to working vars */
+  /*
+  a = state[0];
+  b = state[1];
+  c = state[2];
+  d = state[3];
+  e = state[4];
+  */
+
+  /* 4 rounds of 20 operations each. Loop unrolled. */
+  if( 1 == *(unsigned char*)&one ){
+    Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3);
+    Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7);
+    Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11);
+    Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15);
+  }else{
+    Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3);
+    Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7);
+    Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11);
+    Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15);
+  }
+  R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19);
+  R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23);
+  R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27);
+  R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31);
+  R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35);
+  R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39);
+  R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43);
+  R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47);
+  R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51);
+  R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55);
+  R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59);
+  R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63);
+  R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67);
+  R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71);
+  R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75);
+  R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79);
+
+  /* Add the working vars back into context.state[] */
+  state[0] += a;
+  state[1] += b;
+  state[2] += c;
+  state[3] += d;
+  state[4] += e;
+}
+
+
+/* Initialize the SHA1 hash */
+static void hash_init(void){
+  /* SHA1 initialization constants */
+  g.cx.state[0] = 0x67452301;
+  g.cx.state[1] = 0xEFCDAB89;
+  g.cx.state[2] = 0x98BADCFE;
+  g.cx.state[3] = 0x10325476;
+  g.cx.state[4] = 0xC3D2E1F0;
+  g.cx.count[0] = g.cx.count[1] = 0;
+}
+
+/* Add new content to the SHA1 hash */
+static void hash_step(const unsigned char *data,  unsigned int len){
+  unsigned int i, j;
+
+  j = g.cx.count[0];
+  if( (g.cx.count[0] += len << 3) < j ){
+    g.cx.count[1] += (len>>29)+1;
+  }
+  j = (j >> 3) & 63;
+  if( (j + len) > 63 ){
+    (void)memcpy(&g.cx.buffer[j], data, (i = 64-j));
+    SHA1Transform(g.cx.state, g.cx.buffer);
+    for(; i + 63 < len; i += 64){
+      SHA1Transform(g.cx.state, &data[i]);
+    }
+    j = 0;
+  }else{
+    i = 0;
+  }
+  (void)memcpy(&g.cx.buffer[j], &data[i], len - i);
+}
+
+
+/* Add padding and compute and output the message digest. */
+static void hash_finish(const char *zName){
+  unsigned int i;
+  unsigned char finalcount[8];
+  unsigned char digest[20];
+  static const char zEncode[] = "0123456789abcdef";
+  char zOut[41];
+
+  for (i = 0; i < 8; i++){
+    finalcount[i] = (unsigned char)((g.cx.count[(i >= 4 ? 0 : 1)]
+       >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */
+  }
+  hash_step((const unsigned char *)"\200", 1);
+  while ((g.cx.count[0] & 504) != 448){
+    hash_step((const unsigned char *)"\0", 1);
+  }
+  hash_step(finalcount, 8);  /* Should cause a SHA1Transform() */
+  for (i = 0; i < 20; i++){
+    digest[i] = (unsigned char)((g.cx.state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
+  }
+  for(i=0; i<20; i++){
+    zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
+    zOut[i*2+1] = zEncode[digest[i] & 0xf];
+  }
+  zOut[i*2]= 0;
+  printf("%s %s\n", zOut, zName);
+}
+/* End of the hashing logic
+*******************************************************************************/
+  
+/*
+** Print an error resulting from faulting command-line arguments and
+** abort the program.
+*/
+static void cmdlineError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0);
+  exit(1);
+}
+
+/*
+** Print an error message for an error that occurs at runtime, then
+** abort the program.
+*/
+static void runtimeError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n");
+  exit(1);
+}
+
+/*
+** Prepare a new SQL statement.  Print an error and abort if anything
+** goes wrong.
+*/
+static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){
+  char *zSql;
+  int rc;
+  sqlite3_stmt *pStmt;
+
+  zSql = sqlite3_vmprintf(zFormat, ap);
+  if( zSql==0 ) runtimeError("out of memory");
+  rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0);
+  if( rc ){
+    runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db),
+                 zSql);
+  }
+  sqlite3_free(zSql);
+  return pStmt;
+}
+static sqlite3_stmt *db_prepare(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  return pStmt;
+}
+
+/*
+** Compute the hash for all rows of the query formed from the printf-style
+** zFormat and its argument.
+*/
+static void hash_one_query(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;        /* The query defined by zFormat and "..." */
+  int nCol;                   /* Number of columns in the result set */
+  int i;                      /* Loop counter */
+
+  /* Prepare the query defined by zFormat and "..." */
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  nCol = sqlite3_column_count(pStmt);
+
+  /* Compute a hash over the result of the query */
+  while( SQLITE_ROW==sqlite3_step(pStmt) ){
+    for(i=0; i<nCol; i++){
+      switch( sqlite3_column_type(pStmt,i) ){
+        case SQLITE_NULL: {
+          hash_step((const unsigned char*)"0",1);
+          if( g.fDebug & DEBUG_FULLTRACE ) fprintf(stderr, "NULL\n");
+          break;
+        }
+        case SQLITE_INTEGER: {
+          sqlite3_uint64 u;
+          int j;
+          unsigned char x[8];
+          sqlite3_int64 v = sqlite3_column_int64(pStmt,i);
+          memcpy(&u, &v, 8);
+          for(j=7; j>=0; j--){
+            x[j] = u & 0xff;
+            u >>= 8;
+          }
+          hash_step((const unsigned char*)"1",1);
+          hash_step(x,8);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "INT %s\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_FLOAT: {
+          sqlite3_uint64 u;
+          int j;
+          unsigned char x[8];
+          double r = sqlite3_column_double(pStmt,i);
+          memcpy(&u, &r, 8);
+          for(j=7; j>=0; j--){
+            x[j] = u & 0xff;
+            u >>= 8;
+          }
+          hash_step((const unsigned char*)"2",1);
+          hash_step(x,8);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "FLOAT %s\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_TEXT: {
+          int n = sqlite3_column_bytes(pStmt, i);
+          const unsigned char *z = sqlite3_column_text(pStmt, i);
+          hash_step((const unsigned char*)"3", 1);
+          hash_step(z, n);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "TEXT '%s'\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_BLOB: {
+          int n = sqlite3_column_bytes(pStmt, i);
+          const unsigned char *z = sqlite3_column_blob(pStmt, i);
+          hash_step((const unsigned char*)"4", 1);
+          hash_step(z, n);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "BLOB (%d bytes)\n", n);
+          }
+          break;
+        }
+      }
+    }
+  }
+  sqlite3_finalize(pStmt);
+}
+
+
+/*
+** Print sketchy documentation for this utility program
+*/
+static void showHelp(void){
+  printf("Usage: %s [options] FILE ...\n", g.zArgv0);
+  printf(
+"Compute a SHA1 hash on the content of database FILE.  System tables such as\n"
+"sqlite_stat1, sqlite_stat4, and sqlite_sequence are omitted from the hash.\n"
+"Options:\n"
+"   --debug N           Set debugging flags to N (experts only)\n"
+"   --like PATTERN      Only hash tables whose name is LIKE the pattern\n"
+"   --schema-only       Only hash the schema - omit table content\n"
+"   --without-schema    Only hash table content - omit the schema\n"
+  );
+}
+
+int main(int argc, char **argv){
+  const char *zDb = 0;         /* Name of the database currently being hashed */
+  int i;                       /* Loop counter */
+  int rc;                      /* Subroutine return code */
+  char *zErrMsg;               /* Error message when opening database */
+  sqlite3_stmt *pStmt;         /* An SQLite query */
+  const char *zLike = 0;       /* LIKE pattern of tables to hash */
+  int omitSchema = 0;          /* True to compute hash on content only */
+  int omitContent = 0;         /* True to compute hash on schema only */
+  int nFile = 0;               /* Number of input filenames seen */
+
+  g.zArgv0 = argv[0];
+  sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
+  for(i=1; i<argc; i++){
+    const char *z = argv[i];
+    if( z[0]=='-' ){
+      z++;
+      if( z[0]=='-' ) z++;
+      if( strcmp(z,"debug")==0 ){
+        if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
+        g.fDebug = strtol(argv[++i], 0, 0);
+      }else
+      if( strcmp(z,"help")==0 ){
+        showHelp();
+        return 0;
+      }else
+      if( strcmp(z,"like")==0 ){
+        if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
+        if( zLike!=0 ) cmdlineError("only one --like allowed");
+        zLike = argv[++i];
+      }else
+      if( strcmp(z,"schema-only")==0 ){
+        omitContent = 1;
+      }else
+      if( strcmp(z,"without-schema")==0 ){
+        omitSchema = 1;
+      }else
+      {
+        cmdlineError("unknown option: %s", argv[i]);
+      }
+    }else{
+      nFile++;
+      if( nFile<i ) argv[nFile] = argv[i];
+    }
+  }
+  if( nFile==0 ){
+    cmdlineError("no input files specified - nothing to do");
+  }
+  if( omitSchema && omitContent ){
+    cmdlineError("only one of --without-schema and --omit-schema allowed");
+  }
+  if( zLike==0 ) zLike = "%";
+
+  for(i=1; i<=nFile; i++){
+    static const int openFlags = 
+       SQLITE_OPEN_READWRITE |     /* Read/write so hot journals can recover */
+       SQLITE_OPEN_URI
+    ;
+    zDb = argv[i];
+    rc = sqlite3_open_v2(zDb, &g.db, openFlags, 0);
+    if( rc ){
+      fprintf(stderr, "cannot open database file '%s'\n", zDb);
+      continue;
+    }
+    rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_master", 0, 0, &zErrMsg);
+    if( rc || zErrMsg ){
+      sqlite3_close(g.db);
+      g.db = 0;
+      fprintf(stderr, "'%s' is not a valid SQLite database\n", zDb);
+      continue;
+    }
+
+    /* Start the hash */
+    hash_init();
+  
+    /* Hash table content */
+    if( !omitContent ){
+      pStmt = db_prepare(
+        "SELECT name FROM sqlite_master\n"
+        " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+        "   AND name NOT LIKE 'sqlite_%%'\n"
+        "   AND name LIKE '%q'\n"
+        " ORDER BY name COLLATE nocase;\n",
+        zLike
+      );
+      while( SQLITE_ROW==sqlite3_step(pStmt) ){
+        /* We want rows of the table to be hashed in PRIMARY KEY order.
+        ** Technically, an ORDER BY clause is required to guarantee that
+        ** order.  However, though not guaranteed by the documentation, every
+        ** historical version of SQLite has always output rows in PRIMARY KEY
+        ** order when there is no WHERE or GROUP BY clause, so the ORDER BY
+        ** can be safely omitted. */
+        hash_one_query("SELECT * FROM \"%w\"", sqlite3_column_text(pStmt,0));
+      }
+      sqlite3_finalize(pStmt);
+    }
+  
+    /* Hash the database schema */
+    if( !omitSchema ){
+      hash_one_query(
+         "SELECT type, name, tbl_name, sql FROM sqlite_master\n"
+         " WHERE tbl_name LIKE '%q'\n"
+         " ORDER BY name COLLATE nocase;\n",
+         zLike
+      );
+    }
+  
+    /* Finish and output the hash and close the database connection. */
+    hash_finish(zDb);
+    sqlite3_close(g.db);
+  }
+  return 0;
+}
@@ -339,6 +339,9 @@ void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){
      pParser->yystack = &pParser->yystk0;
      pParser->yystksz = 1;
    }
+#endif
+#ifndef YYNOERRORRECOVERY
+    pParser->yyerrcnt = -1;
 #endif
    pParser->yytos = pParser->yystack;
    pParser->yystack[0].stateno = 0;
@@ -598,6 +601,9 @@ static void yy_shift(
    }
  }
 #endif
+  if( yyNewState > YY_MAX_SHIFT ){
+    yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+  }
  yytos = yypParser->yytos;
  yytos->stateno = (YYACTIONTYPE)yyNewState;
  yytos->major = (YYCODETYPE)yyMajor;
@@ -810,9 +816,6 @@ void Parse(
  do{
    yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
    if( yyact <= YY_MAX_SHIFTREDUCE ){
-      if( yyact > YY_MAX_SHIFT ){
-        yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
-      }
      yy_shift(yypParser,yyact,yymajor,yyminor);
 #ifndef YYNOERRORRECOVERY
      yypParser->yyerrcnt--;
@@ -33,6 +33,7 @@ struct GlobalVars {
  const char *zArgv0;       /* Name of program */
  int bSchemaOnly;          /* Only show schema differences */
  int bSchemaPK;            /* Use the schema-defined PK, not the true PK */
+  int bHandleVtab;          /* Handle fts3, fts4, fts5 and rtree vtabs */
  unsigned fDebug;          /* Debug flags */
  sqlite3 *db;              /* The database connection */
 } g;
@@ -1735,6 +1736,144 @@ end_changeset_one_table:
  sqlite3_free(zId);
 }

+/*
+** Extract the next SQL keyword or quoted string from buffer zIn and copy it
+** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
+** Return a pointer to the character within zIn immediately following 
+** the token or quoted string just extracted.
+*/
+const char *gobble_token(const char *zIn, char *zBuf, int nBuf){
+  const char *p = zIn;
+  char *pOut = zBuf;
+  char *pEnd = &pOut[nBuf-1];
+  char q = 0;                     /* quote character, if any */
+
+  if( p==0 ) return 0;
+  while( *p==' ' ) p++;
+  switch( *p ){
+    case '"': q = '"'; break;
+    case '\'': q = '\''; break;
+    case '`': q = '`'; break;
+    case '[': q = ']'; break;
+  }
+
+  if( q ){
+    p++;
+    while( *p && pOut<pEnd ){
+      if( *p==q ){
+        p++;
+        if( *p!=q ) break;
+      }
+      if( pOut<pEnd ) *pOut++ = *p;
+      p++;
+    }
+  }else{
+    while( *p && *p!=' ' && *p!='(' ){
+      if( pOut<pEnd ) *pOut++ = *p;
+      p++;
+    }
+  }
+
+  *pOut = '\0';
+  return p;
+}
+
+/*
+** This function is the implementation of SQL scalar function "module_name":
+**
+**   module_name(SQL)
+**
+** The only argument should be an SQL statement of the type that may appear
+** in the sqlite_master table. If the statement is a "CREATE VIRTUAL TABLE"
+** statement, then the value returned is the name of the module that it
+** uses. Otherwise, if the statement is not a CVT, NULL is returned.
+*/
+static void module_name_func(
+  sqlite3_context *pCtx, 
+  int nVal, sqlite3_value **apVal
+){
+  const char *zSql;
+  char zToken[32];
+
+  assert( nVal==1 );
+  zSql = (const char*)sqlite3_value_text(apVal[0]);
+
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "create") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "virtual") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "table") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "using") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  
+  sqlite3_result_text(pCtx, zToken, -1, SQLITE_TRANSIENT);
+}
+
+/*
+** Return the text of an SQL statement that itself returns the list of
+** tables to process within the database.
+*/
+const char *all_tables_sql(){
+  if( g.bHandleVtab ){
+    int rc;
+  
+    rc = sqlite3_exec(g.db, 
+        "CREATE TEMP TABLE tblmap(module, postfix);"
+        "INSERT INTO temp.tblmap VALUES"
+        "('fts3', '_content'), ('fts3', '_segments'), ('fts3', '_segdir'),"
+  
+        "('fts4', '_content'), ('fts4', '_segments'), ('fts4', '_segdir'),"
+        "('fts4', '_docsize'), ('fts4', '_stat'),"
+  
+        "('fts5', '_data'), ('fts5', '_idx'), ('fts5', '_content'),"
+        "('fts5', '_docsize'), ('fts5', '_config'),"
+  
+        "('rtree', '_node'), ('rtree', '_rowid'), ('rtree', '_parent');"
+        , 0, 0, 0
+    );
+    assert( rc==SQLITE_OK );
+  
+    rc = sqlite3_create_function(
+        g.db, "module_name", 1, SQLITE_UTF8, 0, module_name_func, 0, 0
+    );
+    assert( rc==SQLITE_OK );
+  
+    return 
+      "SELECT name FROM main.sqlite_master\n"
+      " WHERE type='table' AND (\n"
+      "    module_name(sql) IS NULL OR \n"
+      "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
+      " ) AND name NOT IN (\n"
+      "  SELECT a.name || b.postfix \n"
+        "FROM main.sqlite_master AS a, temp.tblmap AS b \n"
+        "WHERE module_name(a.sql) = b.module\n" 
+      " )\n"
+      "UNION \n"
+      "SELECT name FROM aux.sqlite_master\n"
+      " WHERE type='table' AND (\n"
+      "    module_name(sql) IS NULL OR \n"
+      "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
+      " ) AND name NOT IN (\n"
+      "  SELECT a.name || b.postfix \n"
+        "FROM aux.sqlite_master AS a, temp.tblmap AS b \n"
+        "WHERE module_name(a.sql) = b.module\n" 
+      " )\n"
+      " ORDER BY name";
+  }else{
+    return
+      "SELECT name FROM main.sqlite_master\n"
+      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+      " UNION\n"
+      "SELECT name FROM aux.sqlite_master\n"
+      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+      " ORDER BY name";
+  }
+}
+
 /*
 ** Print sketchy documentation for this utility program
 */
@@ -1751,6 +1890,7 @@ static void showHelp(void){
 "  --summary             Show only a summary of the differences\n"
 "  --table TAB           Show only differences in table TAB\n"
 "  --transaction         Show SQL output inside a transaction\n"
+"  --vtab                Handle fts3, fts4, fts5 and rtree tables\n"
  );
 }

@@ -1821,6 +1961,9 @@ int main(int argc, char **argv){
      if( strcmp(z,"transaction")==0 ){
        useTransaction = 1;
      }else
+      if( strcmp(z,"vtab")==0 ){
+        g.bHandleVtab = 1;
+      }else
      {
        cmdlineError("unknown option: %s", argv[i]);
      }
@@ -1875,14 +2018,7 @@ int main(int argc, char **argv){
    xDiff(zTab, out);
  }else{
    /* Handle tables one by one */
-    pStmt = db_prepare(
-      "SELECT name FROM main.sqlite_master\n"
-      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
-      " UNION\n"
-      "SELECT name FROM aux.sqlite_master\n"
-      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
-      " ORDER BY name"
-    );
+    pStmt = db_prepare( all_tables_sql() );
    while( SQLITE_ROW==sqlite3_step(pStmt) ){
      xDiff((const char*)sqlite3_column_text(pStmt,0), out);
    }
Author	SHA1	Message	Date
dan	e336b8914c	Add the sqlite3rbu_state() API. Used to determine the current state (creating OAL, ready to move OAL, incremental-checkpoint, finished or error) of an RBU operation. FossilOrigin-Name: 92e7df0ff5c4c118c63d92a767dc82700438a310	2016-07-01 12:39:58 +00:00
drh	05d1bad674	Prevent the WhereLoop.rSetup cost estimate from going negative on complex queries. FossilOrigin-Name: f81050859170c8708a1b296da8dd3ef0dd314a11	2016-06-26 04:06:28 +00:00
drh	4b37cd49a6	Fix the handling of OP_Eq opcodes that compare a register against itself and that require an affinity change. FossilOrigin-Name: 507014e4c7a70cd09410c89c8ed466c8edab39d2	2016-06-25 11:43:47 +00:00
drh	f5fe00399f	Fix the ctime.test script so that it works with clang. FossilOrigin-Name: 77e4f7a36e6e0ebe842bcb36b2557a5bfba90d3f	2016-06-24 06:23:13 +00:00
drh	be11ade7cf	Fix the Makefile.in to avoid unnecessary recompiles of sqlite3.c. FossilOrigin-Name: 67c39e16442d9359b24d60d5f97bd66c19eff16a	2016-06-24 02:50:44 +00:00
dan	b10ab53f3a	Update the sqldiff utility so that if the --vtab switch is specified "rtree", "fts3", "fts4" and "fts5" tables are diff'd directly and the underlying real database tables ignored. Without this switch, all virtual tables are ignored and the diff is performed on the underlying real tables. FossilOrigin-Name: b8671e9434180878cfe15d70b793fdee69aabccf	2016-06-23 16:48:35 +00:00
drh	ec5935924d	Fix the build for -DSQLITE_OMIT_VIRTUALTABLE FossilOrigin-Name: 911131424857430d46784b167399ecf192584ca2	2016-06-23 12:35:04 +00:00
dan	9c987a833c	Update the sqldiff utility so that if the --vtab switch is specified "rtree", "fts3", "fts4" and "fts5" tables are diff'd directly and the underlying real database tables ignored. Without this switch, all virtual tables are ignored and the diff is performed on the underlying real tables. FossilOrigin-Name: 5d0a9d4c45730e47576bc77373fa7075a74051a5	2016-06-21 10:34:41 +00:00
dan	ee84b0f690	For a table on the rhs of a LEFT JOIN operator, do not include terms like "IS NULL" from the WHERE clause in the cursor-hint. These may be false for rows that the cursor would otherwise visit, but true for a row of all NULL values generated by the LEFT JOIN. FossilOrigin-Name: 913e595615e2ef40fb431f6e7678f6fc8439782e	2016-06-20 17:25:50 +00:00
dan	2b693d63e4	Allow LIKE operators that appear in a WHERE clause to be included in the cursor-hint for a cursor on the rhs of a LEFT JOIN. FossilOrigin-Name: 7455d932f5079ffe40462a8c119fc22b8a9bcbcc	2016-06-20 17:22:06 +00:00
dan	e6912fd819	Include WHERE terms in the cursor-hint passed to a cursor opened for the rhs of a LEFT JOIN iff we can be sure that those terms will not evaluate to true if the LEFT JOIN generates a row of NULLs. FossilOrigin-Name: 998095aba01b75f685ed981b377e1dfe650d9bbf	2016-06-17 19:27:13 +00:00
dan	c497441406	Fix a duplicate test name in cursorhint2.test. FossilOrigin-Name: fcd12b69cee1335224a65aa6d22c4e302b889398	2016-06-17 14:59:40 +00:00
dan	419b03c1bb	Fix a typo in the cursorhint2.test script. FossilOrigin-Name: c1a5a57cf10dca91082963dcbd3e3ffebc3707ef	2016-06-17 14:47:37 +00:00
dan	b324cf756e	If a table is on the rhs of a LEFT JOIN, include only terms from the joins ON(...) clause in the cursor-hint passed via OP_CursorHint. FossilOrigin-Name: 2a2346b04235c6d0b7a8e64c92ee31018285c29f	2016-06-17 14:33:32 +00:00
dan	3480bfdae9	Add a missing OP_ColumnsUsed opcode to code for expressions like "? IN (SELECT ...)" in cases where expression can use an index that may contain NULL values. FossilOrigin-Name: 0b1579caf06a2c42433b8bc9dc28c9ad381aa07c	2016-06-16 17:14:02 +00:00
drh	6b92288119	Fix a typo in a comment on the SrcList object. FossilOrigin-Name: 48b555c42de1cbc031fb6c2c93ef170e491c7d76	2016-06-16 11:16:53 +00:00
drh	c14566aff3	Fix RBU so that it builds with -DSQLITE_ENABLE_8_3_NAMES. Fix "PRAGMA compile_options" for SQLITE_ENABLE_8_3_NAMES such that it reports the numeric setting: "1" or "2". FossilOrigin-Name: 0230ca17ba20ecd3d213788ad6891973d52a7b72	2016-06-13 19:58:46 +00:00
drh	6c0e41b951	Fix the declaration of the table implemented by DBSTAT so that it uses correct datatypes. FossilOrigin-Name: a1e1cdc51d1c68502f43ac72c28ba87cb1916a0d	2016-06-13 15:59:37 +00:00
drh	debaa86c1c	Fix an incorrect assert() in the btree logic. FossilOrigin-Name: fcf6114be94b260641e7c78a58db16a31ac5ab35	2016-06-13 12:51:20 +00:00
drh	edc4024b9d	Fix the "onecolumn" and "exists" methods of the TCL interface so that they work in combination with the "profile" callback. FossilOrigin-Name: d362ba157f993fc74a590cf15a9a2fa589278dd7	2016-06-13 12:34:38 +00:00
drh	4d249e6128	Enhance "PRAGMA table_info" to that it provides information about eponymous virtual tables. FossilOrigin-Name: 53a1e5d51304cb3de700c1807a2c945a40240576	2016-06-10 22:49:01 +00:00
drh	8dc570b6af	Prefer to use partial indexes for full table scans when that is possible. FossilOrigin-Name: fe1874321ba31cec9ae65387920c33d8d0178ed8	2016-06-08 18:07:21 +00:00
drh	b9f3f6b672	Add the "dbhash.exe" utility program that computes a SHA1 hash over the invariant content of an SQLite database file. Free space in the file, the page size, auto_vacuum status, text encoding, and so forth do not change the hash. Only the content matters. FossilOrigin-Name: f48a4ad33ecd4a86f5529596ff11829ba38b0875	2016-06-08 14:04:50 +00:00
drh	1d5721a777	Fix an undersized buffer in the SHA1 implementation. FossilOrigin-Name: fb2768154c513881886e89801e906bea959197b3	2016-06-08 13:59:35 +00:00
drh	7c4942cb1a	Fix the dbhash utility so that it ignores the root page number when hashing the sqlite_master table. Add new command-line options. Add the ability to hash multiple databases with a single command. FossilOrigin-Name: 44f157e0f0d5a76ef9002b2592164c4fdae89e34	2016-06-08 13:49:28 +00:00
drh	290fcaa2d1	An initial attempt at a "dbhash" command-line utility. FossilOrigin-Name: 2247649ca215c06205b33b2250eb809baf39263a	2016-06-08 01:03:05 +00:00
drh	a83a5c3fbc	Fix the walcrash4.test test module so that it works on windows. FossilOrigin-Name: 2091a4c9231c7871f27661adc27dd7df26500f6c	2016-06-07 20:25:19 +00:00
mistachkin	4a58d5fdf5	In winFullPathname, translate '/X:' to 'X:' before doing anything else. FossilOrigin-Name: e404ad705d0e2d96025d05cc88348ffcd0703533	2016-06-06 20:36:26 +00:00
drh	8a5fd26116	Translate filenames of the form "/c:/" into just "c:/" on WinRT and Cygwin. (SQLite has long done this for Win32/WinNT.) FossilOrigin-Name: f8470ffc49918099820cc0008b9089e5fe8a7dff	2016-06-06 20:27:15 +00:00
drh	0efd37f28b	Fix lempar.c so that the shift-reduce optimization works for error processing. This is a Lemon issue only and has no impact on SQLite. FossilOrigin-Name: 3665a2f554e5b8d2717d952dbaf325a39c637c31	2016-06-06 18:17:36 +00:00
drh	4335ad05b8	Initialize the yyerrcnt variable in the lemon parser template. This has no effect on SQLite itself. FossilOrigin-Name: 45531654f7f5be3a08e9ee8598f14efe028245d8	2016-06-06 13:24:57 +00:00
drh	05a360913d	Small performance boost and size decrease in sqlite3BtreeMovetoUnpacked(). FossilOrigin-Name: e106a77d85c20ae23ebe49a5acceeaffecb40fc2	2016-06-06 01:54:20 +00:00
drh	c80937a5f5	Small performance improvement in the LIKE function. FossilOrigin-Name: 5fb0c35459cf7a8662bf8cd84ac345f6fafda6cc	2016-06-06 01:48:14 +00:00
drh	5cd22006ef	Btree interface refactoring: (1) The sqlite3BtreeKeySize() interface is renamed to sqlite3BtreeIntegerKey() and modified to work only for table btrees with a rowid. (2) The sqlite3BtreeDataSize() interface is renamed to sqlite3BtreePayloadSize() and modified to work with any btree. (3) The sqlite3BtreeDataFetch() and sqlite3BtreeKeyFetch() routines are combined into a single sqlite3BtreePayloadFetch() routine. The result of these changes is a smaller binary and fewer CPU cycles needed to run queries. FossilOrigin-Name: 2d831074cf730dce47de5880d7b4570d04d15fee	2016-06-06 01:14:08 +00:00
drh	5e08d0fcb6	Improved comment on cursorOwnsBtShared(). No changes to code. FossilOrigin-Name: 5e269c2d2667df65592a32d6bc2fa388fd7eb181	2016-06-04 21:05:54 +00:00
drh	d66c4f809d	Fix a C99-ism and a compiler warning for MSVC. FossilOrigin-Name: aa53a36ea2eb90cc4939e37e6ad320b4211692fd	2016-06-04 20:58:35 +00:00
drh	a7c90c42ea	Change the sqlite3BtreeKeySize() interface into sqlite3BtreeIntegerKey() and make it only work for table btrees. Change sqlite3BtreeDataSize() into sqlite3BtreePayloadSize() and make it work for all btrees. Combine sqlite3BtreeDataFetch() and sqlite3BtreeKeyFetch() into a single sqlite3BtreePayloadFetch() routine. These changes seem to make the b-tree interface more rational and they reduce both binary size and CPU usage. FossilOrigin-Name: bef35e18dd19732f7859287b097feeb593e5900f	2016-06-04 20:37:10 +00:00
drh	f94fdd832c	Allocate KeyInfo objects from lookaside if possible. FossilOrigin-Name: b411107a3609d53af4e147f01e311b858b78420b	2016-06-04 17:12:26 +00:00
drh	30e3fdf0ee	Fix up speedtest1.c so that it will compile and run on SQLite versions prior to 3.6.18 (circa 2009-09-11). FossilOrigin-Name: 9583c0fb39f357a76e0c99ea03b034aea3e03c75	2016-06-04 16:33:48 +00:00
drh	cc97173810	Remove an unreachable branch in the UNIQUE constraint parsing. FossilOrigin-Name: 313e990c741d7db082f1d9b9f3f4c65fdd91b812	2016-06-04 13:57:41 +00:00
drh	f0755fc85e	The OR optimization is usable on virtual tables with LIKE, REGEXP and/or GLOB terms in the WHERE clause. FossilOrigin-Name: fa3a89fc0b88d5ad7f5c232198847b3483eef611	2016-06-03 18:59:37 +00:00
drh	a4680a8b19	Add support for virtual tables using a WITHOUT ROWID schema. This merge also includes enhancements to the CSV extension, which is used for testing of the new WITHOUT ROWID virtual table mechanism. FossilOrigin-Name: aa7e9d0cc15c050205b09d5a50f985e4ade571d0	2016-06-03 18:44:43 +00:00
drh	0b2c140a4d	Fix a memory leak when a WITHOUT ROWID eponymous virtual table is used. FossilOrigin-Name: 31b83a7d7e553163eb186fc966a885f237554ec2	2016-06-03 18:21:04 +00:00
drh	7edcf627af	Disallow access to the rowid column on WITHOUT ROWID virtual tables. FossilOrigin-Name: d31c25972bfb6d04caad05534505698776e7e6d5	2016-06-03 17:27:14 +00:00
drh	c576b27580	Enhance the sqlite3_load_extension() interface to permit extensions to return SQLITE_OK_LOAD_PERMANENTLY which will prevents unloading when the database connection closes. FossilOrigin-Name: 5908aa4dc59925c414b236b6fd3d3ecd7f3da435	2016-06-03 13:35:55 +00:00
drh	ac9c3d2c18	Performance optimizations on the CSV virtual table. Disallow WITHOUT ROWID virtual tables that have an xUpdate method, for now. FossilOrigin-Name: 3134b3266c36c9d018e8d365ef46ef638c0792f4	2016-06-03 01:01:57 +00:00
drh	35db31b24b	Add the CSV extension to the test fixture. Fix a memory leak in the CSV extension. Add test cases for the CSV extension, including one that uses a WITHOUT ROWID virtual table participating in the OR optimization. FossilOrigin-Name: 95f483e86e30ae68108904400e18ed41d389446b	2016-06-02 23:13:21 +00:00
drh	adcba64df4	Add the data= parameter to the CSV virtual table extension. FossilOrigin-Name: 769191042aa14e6eccdfe2391fc1011171d5c9ad	2016-06-02 17:44:24 +00:00
drh	273bfe9f20	Fix corner cases in the WITHOUT ROWID virtual table logic. FossilOrigin-Name: a393bbb972660c1ffcdda923d0f3564ecfcd2f0a	2016-06-02 16:22:53 +00:00
dan	82a1c0e5bb	Fix an issue preventing RBU vacuum from working with virtual tables. FossilOrigin-Name: 3bd85fa5a9a489fd505c973e37c33a76c1b0e957	2016-06-01 10:37:50 +00:00
mistachkin	9f6e686f59	Fix compilation issues with the VFS stat extension. FossilOrigin-Name: f6e956525b03fa07190e5521edac4758c386cc80	2016-06-01 05:02:05 +00:00
drh	62340f8498	An experimental branch with code that allows virtual tables to be declared as WITHOUT ROWID tables. This might be useful for virtual tables that model external data sources that do not have a convenient way of computing a unique rowid. The current check-in almost works, but there are still serious issues. FossilOrigin-Name: 49638f180e26477974cacc69b79e0be0a5e18b29	2016-05-31 21:18:15 +00:00
drh	1fc1a0f280	Add the columns=N parameter to the CSV extension. FossilOrigin-Name: 28ebeadd6a4c9ff2ce9fc86a0f0fe2f6cf94d3ac	2016-05-31 18:44:33 +00:00
drh	abfd272b59	Add the testflags parameter to the csv extension. FossilOrigin-Name: b93fb2fe0df1b3bea2bc2a4e1528da74ab290593	2016-05-31 18:08:35 +00:00
drh	724b18966d	Add the "csv" virtual table for reading CSV files, as an extension in the ext/misc/ subfolder. FossilOrigin-Name: 00d3570c8bb96469c984903e20de589e998d4636	2016-05-31 16:22:48 +00:00
dan	0df5024e83	Fix an FTS5 problem (segfault or incorrect query results) with "... MATCH 'x OR y' ORDER BY rank" queries when either token 'x' or 'y' is completely absent from the dataset. FossilOrigin-Name: 64ca1a835a89fd211078d2cd8f9b649e89be528d	2016-05-30 08:28:21 +00:00
dan	dbd2dcbd5e	Experimental change to allow virtual tables to take advantage of LIKE, REGEXP and GLOB terms that are part of OR expressions within WHERE clauses. FossilOrigin-Name: 242507b4ff96bc4c7c7844dbe1c2b8508dbf1d01	2016-05-28 18:53:55 +00:00
drh	66bf810467	Remove an unnecessary malloc from the vfsstat extension. FossilOrigin-Name: 24f258c2392290168cf34622b89a4a406a3dd853	2016-05-28 17:45:15 +00:00
drh	c1502e2f4f	Enhance the sqlite3_load_extension() interface to permit extensions to return SQLITE_OK_LOAD_PERMANENTLY which will prevent the extensions from unloading when the database connection closes. FossilOrigin-Name: d3f99a5e8d0486d0917dfe96987c179a6f63b850	2016-05-28 17:23:08 +00:00
drh	444c445ff8	Enhance "PRAGMA compile_options" so that it shows the version of the compiler used to generate the executable, for common compilers. FossilOrigin-Name: 6a0f200957ea294a2ae06c0b039a10ac838925f2	2016-05-28 15:22:33 +00:00
dan	00dae0a3ca	Update the amalgamation-tarball configure script so that it can use header file "readline/readline.h" with library file "libedit". FossilOrigin-Name: cbf72b04bb0650b62336d86b22ef59315ccdc183	2016-05-28 15:09:35 +00:00
mistachkin	f35b8f8369	Clang can define _MSC_VER in some circumstances; therefore, check for Clang first. FossilOrigin-Name: 3f710bc3617691fa7432a24f1410d4651ca181b6	2016-05-28 00:13:21 +00:00
mistachkin	7ce6cdd515	Check for Clang before GCC. FossilOrigin-Name: 7c2cd4b05f88a759bdf72d3a249db5984fd7105d	2016-05-27 21:13:43 +00:00
mistachkin	756e09c338	Remove surplus quotation marks from the COMPILER= compile_options setting for Clang. FossilOrigin-Name: 5c966f200018043f5258eb03e043bb25c7766131	2016-05-27 20:30:02 +00:00
drh	2b9672e998	Remove surplus quotation marks from the COMPILER= compile_options setting for GCC. FossilOrigin-Name: 664c132dabe4a871d5c39c11b339f871ca3a5d31	2016-05-27 19:34:29 +00:00
mistachkin	15e2ddeab3	Add basic compiler information to the results of 'PRAGMA compile_options'. FossilOrigin-Name: d734e2df40d95da74ce0c8dd2da1c14033cad8bb	2016-05-27 18:09:45 +00:00