Change the name of the intarray() extension to carray() and give it an

optional third parameter that specifies the datatype as one of 'int32',
'int64', 'double', or 'char*'.  'int32' is the default.

FossilOrigin-Name: a204ba99db34b356acb259189158a32d2df25da0

Makefile.in

@@ -416,7 +416,9 @@ TESTSRC = \
 #
 TESTSRC += \
   $(TOP)/ext/misc/amatch.c \
+  $(TOP)/ext/misc/carray.c \
   $(TOP)/ext/misc/closure.c \
+  $(TOP)/ext/misc/csv.c \
   $(TOP)/ext/misc/eval.c \
   $(TOP)/ext/misc/fileio.c \
   $(TOP)/ext/misc/fuzzer.c \
@@ -538,7 +540,8 @@ TESTPROGS = \
   testfixture$(TEXE) \
   sqlite3$(TEXE) \
   sqlite3_analyzer$(TEXE) \
-  sqldiff$(TEXE)
+  sqldiff$(TEXE) \
+  dbhash$(TEXE)
 
 # Databases containing fuzzer test cases
 #
@@ -586,12 +589,15 @@ sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
 		$(TOP)/src/shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
 
-sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
-	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
+sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
+	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)
 
-scrub$(TEXE):	$(TOP)/ext/misc/scrub.c sqlite3.o
+dbhash$(TEXE):	$(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h
+	$(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS)
+
+scrub$(TEXE):	$(TOP)/ext/misc/scrub.c sqlite3.lo
 	$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
-		$(TOP)/ext/misc/scrub.c sqlite3.o $(TLIBS)
+		$(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS)
 
 srcck1$(BEXE):	$(TOP)/tool/srcck1.c
 	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
@@ -606,8 +612,8 @@ fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
 
-mptester$(TEXE):	sqlite3.c $(TOP)/mptest/mptest.c
-	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
+mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
+	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
 		$(TLIBS) -rpath "$(libdir)"
 
 MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
@@ -1151,8 +1157,8 @@ rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
 LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
 
-wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.c
-	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS)
+wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)
 
 speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
@@ -1169,7 +1175,7 @@ loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
 #
 VALIDIDS=' sqlite3(changeset|changegroup|session)?_'
 checksymbols: sqlite3.lo
-	nm -g --defined-only sqlite3.o | egrep -v $(VALIDIDS); test $$? -ne 0
+	nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0
 	echo '0 errors out of 1 tests'
 
 # Build the amalgamation-autoconf package.  The amalamgation-tarball target builds
@@ -1250,6 +1256,7 @@ clean:
 	rm -f fuzzershell fuzzershell.exe
 	rm -f fuzzcheck fuzzcheck.exe
 	rm -f sqldiff sqldiff.exe
+	rm -f dbhash dbhash.exe
 	rm -f fts5.* fts5parse.*
 
 distclean:	clean

Makefile.msc

@@ -1292,7 +1292,9 @@ TESTSRC = \
 #
 TESTEXT = \
   $(TOP)\ext\misc\amatch.c \
+  $(TOP)\ext\misc\carray.c \
   $(TOP)\ext\misc\closure.c \
+  $(TOP)\ext\misc\csv.c \
   $(TOP)\ext\misc\eval.c \
   $(TOP)\ext\misc\fileio.c \
   $(TOP)\ext\misc\fuzzer.c \
@@ -1377,7 +1379,8 @@ TESTPROGS = \
   testfixture.exe \
   $(SQLITE3EXE) \
   sqlite3_analyzer.exe \
-  sqldiff.exe
+  sqldiff.exe \
+  dbhash.exe
 
 # Databases containing fuzzer test cases
 #
@@ -1455,6 +1458,9 @@ $(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_S
 sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+dbhash.exe:	$(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
@@ -2104,6 +2110,6 @@ clean:
 	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
-	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
+	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
 	del /Q fts5.* fts5parse.* 2>NUL
 # <</mark>>

autoconf/configure.ac

@@ -30,40 +30,52 @@ AC_FUNC_STRERROR_R
 AC_CONFIG_FILES([Makefile sqlite3.pc])
 AC_SUBST(BUILD_CFLAGS)
 
-#-----------------------------------------------------------------------
+#-------------------------------------------------------------------------
+# Two options to enable readline compatible libraries: 
+#
 #   --enable-editline
 #   --enable-readline
 #
-AC_ARG_ENABLE(editline, [AS_HELP_STRING(
-  [--enable-editline], 
-  [use BSD libedit])], 
-  [], [enable_editline=yes])
-AC_ARG_ENABLE(readline, [AS_HELP_STRING(
-  [--enable-readline], 
-  [use readline])], 
-  [], [enable_readline=yes])
-if test x"$enable_editline" != xno ; then
-  sLIBS=$LIBS
-  LIBS=""
-  AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no])
-  READLINE_LIBS=$LIBS
-  if test x"$LIBS" != "x"; then
-    AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
-    enable_readline=no
-  else
-    unset ac_cv_search_readline
-  fi
-  LIBS=$sLIBS
-fi
-if test x"$enable_readline" != xno ; then
-  sLIBS=$LIBS
-  LIBS=""
-  AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], [])
-  AC_SEARCH_LIBS(readline, readline, [], [enable_readline=no])
-  AC_CHECK_FUNCS(readline, [], [])
-  READLINE_LIBS=$LIBS
-  LIBS=$sLIBS
-fi
+# Both are enabled by default. If, after command line processing both are
+# still enabled, the script searches for editline first and automatically
+# disables readline if it is found. So, to use readline explicitly, the
+# user must pass "--disable-editline". To disable command line editing
+# support altogether, "--disable-editline --disable-readline".
+#
+# When searching for either library, check for headers before libraries 
+# as some distros supply packages that contain libraries but not header
+# files, which come as a separate development package.
+#
+AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])])
+AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])])
+
+AS_IF([ test x"$enable_editline" != xno ],[
+  AC_CHECK_HEADERS([editline/readline.h],[
+    sLIBS=$LIBS
+    LIBS=""
+    AC_SEARCH_LIBS([readline],[edit],[
+      AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
+      READLINE_LIBS=$LIBS
+      enable_readline=no
+    ])
+    AS_UNSET(ac_cv_search_readline)
+    LIBS=$sLIBS
+  ])
+])
+
+AS_IF([ test x"$enable_readline" != xno ],[
+  AC_CHECK_HEADERS([readline/readline.h],[
+    sLIBS=$LIBS
+    LIBS=""
+    AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], [])
+    AC_SEARCH_LIBS(readline,[readline edit], [
+      AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper)
+      READLINE_LIBS=$LIBS
+    ])
+    LIBS=$sLIBS
+  ])
+])
+
 AC_SUBST(READLINE_LIBS)
 #-----------------------------------------------------------------------
 

ext/fts5/fts5Int.h

@@ -686,7 +686,6 @@ int sqlite3Fts5ExprPopulatePoslists(
     Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 );
 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
-void sqlite3Fts5ExprClearEof(Fts5Expr*);
 
 int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 

ext/fts5/fts5_expr.c

@@ -2617,17 +2617,6 @@ void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
   fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
 }
 
-static void fts5ExprClearEof(Fts5ExprNode *pNode){
-  int i;
-  for(i=0; i<pNode->nChild; i++){
-    fts5ExprClearEof(pNode->apChild[i]);
-  }
-  pNode->bEof = 0;
-}
-void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
-  fts5ExprClearEof(pExpr->pRoot);
-}
-
 /*
 ** This function is only called for detail=columns tables. 
 */

ext/fts5/fts5_main.c

@@ -1186,7 +1186,6 @@ static int fts5FilterMethod(
     pCsr->ePlan = FTS5_PLAN_SOURCE;
     pCsr->pExpr = pTab->pSortCsr->pExpr;
     rc = fts5CursorFirst(pTab, pCsr, bDesc);
-    sqlite3Fts5ExprClearEof(pCsr->pExpr);
   }else if( pMatch ){
     const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
     if( zExpr==0 ) zExpr = "";

ext/fts5/test/fts5rank.test

@@ -91,9 +91,38 @@ do_test 2.7 {
 } {1 3 2}
 
 
+#--------------------------------------------------------------------------
+# At one point there was a problem with queries such as:
+#
+#   ... MATCH 'x OR y' ORDER BY rank;
+#
+# if there were zero occurrences of token 'y' in the dataset. The
+# following tests verify that that problem has been addressed.
+#
+foreach_detail_mode $::testprefix {
+  do_execsql_test 3.0 {
+    CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%);
+    INSERT INTO y1 VALUES('test xyz');
+    INSERT INTO y1 VALUES('test test xyz test');
+    INSERT INTO y1 VALUES('test test xyz');
+  }
 
+  do_execsql_test 3.1 {
+    SELECT rowid FROM y1('test OR tset');
+  } {1 2 3}
 
+  do_execsql_test 3.2 {
+    SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1)
+  } {2 3 1}
 
+  do_execsql_test 3.3 {
+    SELECT rowid FROM y1('test OR tset') ORDER BY +rank
+  } {2 3 1}
+
+  do_execsql_test 3.4 {
+    SELECT rowid FROM y1('test OR tset') ORDER BY rank
+  } {2 3 1}
+}
 
 
 finish_test

ext/misc/carray.c

@@ -0,0 +1,369 @@
+/*
+** 2016-06-29
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file demonstrates how to create a table-valued-function that
+** returns the values in a C-language array.
+** Examples:
+**
+**      SELECT * FROM array($ptr,5)
+**
+** The query above returns 5 integers contained in a C-language array
+** at the address $ptr.  $ptr is a pointer to the array of integers that
+** has been cast to an integer.
+**
+** There is an optional third parameter to determine the datatype of
+** the C-language array.  Allowed values of the third parameter are
+** 'int32', 'int64', 'double', 'char*'.  Example:
+**
+**      SELECT * FROM array($ptr,10,'char*');
+**
+** HOW IT WORKS
+**
+** The carray "function" is really a virtual table with the
+** following schema:
+**
+**     CREATE TABLE carray(
+**       value,
+**       pointer HIDDEN,
+**       count HIDDEN,
+**       ctype TEXT HIDDEN
+**     );
+**
+** If the hidden columns "pointer" and "count" are unconstrained, then 
+** the virtual table has no rows.  Otherwise, the virtual table interprets
+** the integer value of "pointer" as a pointer to the array and "count"
+** as the number of elements in the array.  The virtual table steps through
+** the array, element by element.
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/*
+** Allowed datatypes
+*/
+#define CARRAY_INT32    0
+#define CARRAY_INT64    1
+#define CARRAY_DOUBLE   2
+#define CARRAY_TEXT     3
+
+/*
+** Names of types
+*/
+static const char *azType[] = { "int32", "int64", "double", "char*" };
+
+
+/* carray_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct carray_cursor carray_cursor;
+struct carray_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  sqlite3_int64 iRowid;      /* The rowid */
+  sqlite3_int64 iPtr;        /* Pointer to array of values */
+  sqlite3_int64 iCnt;        /* Number of integers in the array */
+  unsigned char eType;       /* One of the CARRAY_type values */
+};
+
+/*
+** The carrayConnect() method is invoked to create a new
+** carray_vtab that describes the carray virtual table.
+**
+** Think of this routine as the constructor for carray_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the carray_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against carray will look like.
+*/
+static int carrayConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  sqlite3_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define CARRAY_COLUMN_VALUE   0
+#define CARRAY_COLUMN_POINTER 1
+#define CARRAY_COLUMN_COUNT   2
+#define CARRAY_COLUMN_CTYPE   3
+
+  rc = sqlite3_declare_vtab(db,
+     "CREATE TABLE x(value,pointer hidden,count hidden,ctype hidden)");
+  if( rc==SQLITE_OK ){
+    pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) );
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for carray_cursor objects.
+*/
+static int carrayDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new carray_cursor object.
+*/
+static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  carray_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a carray_cursor.
+*/
+static int carrayClose(sqlite3_vtab_cursor *cur){
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a carray_cursor to its next row of output.
+*/
+static int carrayNext(sqlite3_vtab_cursor *cur){
+  carray_cursor *pCur = (carray_cursor*)cur;
+  pCur->iRowid++;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the carray_cursor
+** is currently pointing.
+*/
+static int carrayColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  carray_cursor *pCur = (carray_cursor*)cur;
+  sqlite3_int64 x = 0;
+  switch( i ){
+    case CARRAY_COLUMN_POINTER:   x = pCur->iPtr;   break;
+    case CARRAY_COLUMN_COUNT:     x = pCur->iCnt;   break;
+    case CARRAY_COLUMN_CTYPE: {
+      sqlite3_result_text(ctx, azType[pCur->eType], -1, SQLITE_STATIC);
+      return SQLITE_OK;
+    }
+    default: {
+      switch( pCur->eType ){
+        case CARRAY_INT32: {
+          int *p = (int*)pCur->iPtr;
+          sqlite3_result_int(ctx, p[pCur->iRowid-1]);
+          return SQLITE_OK;
+        }
+        case CARRAY_INT64: {
+          sqlite3_int64 *p = (sqlite3_int64*)pCur->iPtr;
+          sqlite3_result_int64(ctx, p[pCur->iRowid-1]);
+          return SQLITE_OK;
+        }
+        case CARRAY_DOUBLE: {
+          double *p = (double*)pCur->iPtr;
+          sqlite3_result_double(ctx, p[pCur->iRowid-1]);
+          return SQLITE_OK;
+        }
+        case CARRAY_TEXT: {
+          const char **p = (const char**)pCur->iPtr;
+          sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT);
+          return SQLITE_OK;
+        }
+      }
+    }
+  }
+  sqlite3_result_int64(ctx, x);
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int carrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  carray_cursor *pCur = (carray_cursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int carrayEof(sqlite3_vtab_cursor *cur){
+  carray_cursor *pCur = (carray_cursor*)cur;
+  return pCur->iRowid>pCur->iCnt;
+}
+
+/*
+** This method is called to "rewind" the carray_cursor object back
+** to the first row of output.
+*/
+static int carrayFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  carray_cursor *pCur = (carray_cursor *)pVtabCursor;
+  if( idxNum ){
+    pCur->iPtr = sqlite3_value_int64(argv[0]);
+    pCur->iCnt = sqlite3_value_int64(argv[1]);
+    if( idxNum<3 ){
+      pCur->eType = CARRAY_INT32;
+    }else{
+      int i;
+      const char *zType = (const char*)sqlite3_value_text(argv[2]);
+      for(i=0; i<sizeof(azType)/sizeof(azType[0]); i++){
+        if( sqlite3_stricmp(zType, azType[i])==0 ) break;
+      }
+      if( i>=sizeof(azType)/sizeof(azType[0]) ){
+        pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf(
+          "unknown datatype: %Q", zType);
+        return SQLITE_ERROR;
+      }else{
+        pCur->eType = i;
+      }
+    }
+  }else{
+    pCur->iPtr = 0;
+    pCur->iCnt = 0;
+  }
+  pCur->iRowid = 1;
+  return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the carray virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+**
+** In this implementation idxNum is used to represent the
+** query plan.  idxStr is unused.
+**
+** idxNum is 2 if the pointer= and count= constraints exist,
+** 3 if the ctype= constraint also exists, and is 0 otherwise.
+** If idxNum is 0, then carray becomes an empty table.
+*/
+static int carrayBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;                 /* Loop over constraints */
+  int ptrIdx = -1;       /* Index of the pointer= constraint, or -1 if none */
+  int cntIdx = -1;       /* Index of the count= constraint, or -1 if none */
+  int ctypeIdx = -1;     /* Index of the ctype= constraint, or -1 if none */
+
+  const struct sqlite3_index_constraint *pConstraint;
+  pConstraint = pIdxInfo->aConstraint;
+  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
+    if( pConstraint->usable==0 ) continue;
+    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    switch( pConstraint->iColumn ){
+      case CARRAY_COLUMN_POINTER:
+        ptrIdx = i;
+        break;
+      case CARRAY_COLUMN_COUNT:
+        cntIdx = i;
+        break;
+      case CARRAY_COLUMN_CTYPE:
+        ctypeIdx = i;
+        break;
+    }
+  }
+  if( ptrIdx>=0 && cntIdx>=0 ){
+    pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[ptrIdx].omit = 1;
+    pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2;
+    pIdxInfo->aConstraintUsage[cntIdx].omit = 1;
+    pIdxInfo->estimatedCost = (double)1;
+    pIdxInfo->estimatedRows = (double)100;
+    pIdxInfo->idxNum = 2;
+    if( ctypeIdx>=0 ){
+      pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3;
+      pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1;
+      pIdxInfo->idxNum = 3;
+    }
+  }else{
+    pIdxInfo->estimatedCost = (double)2147483647;
+    pIdxInfo->estimatedRows = (double)2147483647;
+    pIdxInfo->idxNum = 0;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** carray virtual table.
+*/
+static sqlite3_module carrayModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  carrayConnect,             /* xConnect */
+  carrayBestIndex,           /* xBestIndex */
+  carrayDisconnect,          /* xDisconnect */
+  0,                         /* xDestroy */
+  carrayOpen,                /* xOpen - open a cursor */
+  carrayClose,               /* xClose - close a cursor */
+  carrayFilter,              /* xFilter - configure scan constraints */
+  carrayNext,                /* xNext - advance a cursor */
+  carrayEof,                 /* xEof - check for end of scan */
+  carrayColumn,              /* xColumn - read data */
+  carrayRowid,               /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_carray_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if( sqlite3_libversion_number()<3008012 ){
+    *pzErrMsg = sqlite3_mprintf(
+        "carray() requires SQLite 3.8.12 or later");
+    return SQLITE_ERROR;
+  }
+  rc = sqlite3_create_module(db, "carray", &carrayModule, 0);
+#endif
+  return rc;
+}

ext/misc/csv.c

@@ -0,0 +1,861 @@
+/*
+** 2016-05-28
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains the implementation of an SQLite virtual table for
+** reading CSV files.
+**
+** Usage:
+**
+**    .load ./csv
+**    CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
+**    SELECT * FROM csv;
+**
+** The columns are named "c1", "c2", "c3", ... by default.  But the
+** application can define its own CREATE TABLE statement as an additional
+** parameter.  For example:
+**
+**    CREATE VIRTUAL TABLE temp.csv2 USING csv(
+**       filename = "../http.log",
+**       schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)"
+**    );
+**
+** Instead of specifying a file, the text of the CSV can be loaded using
+** the data= parameter.
+**
+** If the columns=N parameter is supplied, then the CSV file is assumed to have
+** N columns.  If the columns parameter is omitted, the CSV file is opened
+** as soon as the virtual table is constructed and the first row of the CSV
+** is read in order to count the tables.
+**
+** Some extra debugging features (used for testing virtual tables) are available
+** if this module is compiled with -DSQLITE_TEST.
+*/
+#include <sqlite3ext.h>
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <stdio.h>
+
+/*
+** A macro to hint to the compiler that a function should not be
+** inlined.
+*/
+#if defined(__GNUC__)
+#  define CSV_NOINLINE  __attribute__((noinline))
+#elif defined(_MSC_VER) && _MSC_VER>=1310
+#  define CSV_NOINLINE  __declspec(noinline)
+#else
+#  define CSV_NOINLINE
+#endif
+
+
+/* Max size of the error message in a CsvReader */
+#define CSV_MXERR 200
+
+/* Size of the CsvReader input buffer */
+#define CSV_INBUFSZ 1024
+
+/* A context object used when read a CSV file. */
+typedef struct CsvReader CsvReader;
+struct CsvReader {
+  FILE *in;              /* Read the CSV text from this input stream */
+  char *z;               /* Accumulated text for a field */
+  int n;                 /* Number of bytes in z */
+  int nAlloc;            /* Space allocated for z[] */
+  int nLine;             /* Current line number */
+  char cTerm;            /* Character that terminated the most recent field */
+  size_t iIn;            /* Next unread character in the input buffer */
+  size_t nIn;            /* Number of characters in the input buffer */
+  char *zIn;             /* The input buffer */
+  char zErr[CSV_MXERR];  /* Error message */
+};
+
+/* Initialize a CsvReader object */
+static void csv_reader_init(CsvReader *p){
+  p->in = 0;
+  p->z = 0;
+  p->n = 0;
+  p->nAlloc = 0;
+  p->nLine = 0;
+  p->nIn = 0;
+  p->zIn = 0;
+  p->zErr[0] = 0;
+}
+
+/* Close and reset a CsvReader object */
+static void csv_reader_reset(CsvReader *p){
+  if( p->in ){
+    fclose(p->in);
+    sqlite3_free(p->zIn);
+  }
+  sqlite3_free(p->z);
+  csv_reader_init(p);
+}
+
+/* Report an error on a CsvReader */
+static void csv_errmsg(CsvReader *p, const char *zFormat, ...){
+  va_list ap;
+  va_start(ap, zFormat);
+  sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap);
+  va_end(ap);
+}
+
+/* Open the file associated with a CsvReader
+** Return the number of errors.
+*/
+static int csv_reader_open(
+  CsvReader *p,               /* The reader to open */
+  const char *zFilename,      /* Read from this filename */
+  const char *zData           /*  ... or use this data */
+){
+  if( zFilename ){
+    p->zIn = sqlite3_malloc( CSV_INBUFSZ );
+    if( p->zIn==0 ){
+      csv_errmsg(p, "out of memory");
+      return 1;
+    }
+    p->in = fopen(zFilename, "rb");
+    if( p->in==0 ){
+      csv_reader_reset(p);
+      csv_errmsg(p, "cannot open '%s' for reading", zFilename);
+      return 1;
+    }
+  }else{
+    assert( p->in==0 );
+    p->zIn = (char*)zData;
+    p->nIn = strlen(zData);
+  }
+  return 0;
+}
+
+/* The input buffer has overflowed.  Refill the input buffer, then
+** return the next character
+*/
+static CSV_NOINLINE int csv_getc_refill(CsvReader *p){
+  size_t got;
+
+  assert( p->iIn>=p->nIn );  /* Only called on an empty input buffer */
+  assert( p->in!=0 );        /* Only called if reading froma file */
+
+  got = fread(p->zIn, 1, CSV_INBUFSZ, p->in);
+  if( got==0 ) return EOF;
+  p->nIn = got;
+  p->iIn = 1;
+  return p->zIn[0];
+}
+
+/* Return the next character of input.  Return EOF at end of input. */
+static int csv_getc(CsvReader *p){
+  if( p->iIn >= p->nIn ){
+    if( p->in!=0 ) return csv_getc_refill(p);
+    return EOF;
+  }
+  return p->zIn[p->iIn++];
+}
+
+/* Increase the size of p->z and append character c to the end. 
+** Return 0 on success and non-zero if there is an OOM error */
+static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){
+  char *zNew;
+  int nNew = p->nAlloc*2 + 100;
+  zNew = sqlite3_realloc64(p->z, nNew);
+  if( zNew ){
+    p->z = zNew;
+    p->nAlloc = nNew;
+    p->z[p->n++] = c;
+    return 0;
+  }else{
+    csv_errmsg(p, "out of memory");
+    return 1;
+  }
+}
+
+/* Append a single character to the CsvReader.z[] array.
+** Return 0 on success and non-zero if there is an OOM error */
+static int csv_append(CsvReader *p, char c){
+  if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c);
+  p->z[p->n++] = c;
+  return 0;
+}
+
+/* Read a single field of CSV text.  Compatible with rfc4180 and extended
+** with the option of having a separator other than ",".
+**
+**   +  Input comes from p->in.
+**   +  Store results in p->z of length p->n.  Space to hold p->z comes
+**      from sqlite3_malloc64().
+**   +  Keep track of the line number in p->nLine.
+**   +  Store the character that terminates the field in p->cTerm.  Store
+**      EOF on end-of-file.
+**
+** Return "" at EOF.  Return 0 on an OOM error.
+*/
+static char *csv_read_one_field(CsvReader *p){
+  int c;
+  p->n = 0;
+  c = csv_getc(p);
+  if( c==EOF ){
+    p->cTerm = EOF;
+    return "";
+  }
+  if( c=='"' ){
+    int pc, ppc;
+    int startLine = p->nLine;
+    pc = ppc = 0;
+    while( 1 ){
+      c = csv_getc(p);
+      if( c<='"' || pc=='"' ){
+        if( c=='\n' ) p->nLine++;
+        if( c=='"' ){
+          if( pc=='"' ){
+            pc = 0;
+            continue;
+          }
+        }
+        if( (c==',' && pc=='"')
+         || (c=='\n' && pc=='"')
+         || (c=='\n' && pc=='\r' && ppc=='"')
+         || (c==EOF && pc=='"')
+        ){
+          do{ p->n--; }while( p->z[p->n]!='"' );
+          p->cTerm = c;
+          break;
+        }
+        if( pc=='"' && c!='\r' ){
+          csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"');
+          break;
+        }
+        if( c==EOF ){
+          csv_errmsg(p, "line %d: unterminated %c-quoted field\n",
+                     startLine, '"');
+          p->cTerm = c;
+          break;
+        }
+      }
+      if( csv_append(p, (char)c) ) return 0;
+      ppc = pc;
+      pc = c;
+    }
+  }else{
+    while( c>',' || (c!=EOF && c!=',' && c!='\n') ){
+      if( csv_append(p, (char)c) ) return 0;
+      c = csv_getc(p);
+    }
+    if( c=='\n' ){
+      p->nLine++;
+      if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--;
+    }
+    p->cTerm = c;
+  }
+  if( p->z ) p->z[p->n] = 0;
+  return p->z;
+}
+
+
+/* Forward references to the various virtual table methods implemented
+** in this file. */
+static int csvtabCreate(sqlite3*, void*, int, const char*const*, 
+                           sqlite3_vtab**,char**);
+static int csvtabConnect(sqlite3*, void*, int, const char*const*, 
+                           sqlite3_vtab**,char**);
+static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*);
+static int csvtabDisconnect(sqlite3_vtab*);
+static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**);
+static int csvtabClose(sqlite3_vtab_cursor*);
+static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr,
+                          int argc, sqlite3_value **argv);
+static int csvtabNext(sqlite3_vtab_cursor*);
+static int csvtabEof(sqlite3_vtab_cursor*);
+static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int);
+static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*);
+
+/* An instance of the CSV virtual table */
+typedef struct CsvTable {
+  sqlite3_vtab base;              /* Base class.  Must be first */
+  char *zFilename;                /* Name of the CSV file */
+  char *zData;                    /* Raw CSV data in lieu of zFilename */
+  long iStart;                    /* Offset to start of data in zFilename */
+  int nCol;                       /* Number of columns in the CSV file */
+  unsigned int tstFlags;          /* Bit values used for testing */
+} CsvTable;
+
+/* Allowed values for tstFlags */
+#define CSVTEST_FIDX  0x0001      /* Pretend that constrained searchs cost less*/
+
+/* A cursor for the CSV virtual table */
+typedef struct CsvCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  CsvReader rdr;                  /* The CsvReader object */
+  char **azVal;                   /* Value of the current row */
+  int *aLen;                      /* Length of each entry */
+  sqlite3_int64 iRowid;           /* The current rowid.  Negative for EOF */
+} CsvCursor;
+
+/* Transfer error message text from a reader into a CsvTable */
+static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){
+  sqlite3_free(pTab->base.zErrMsg);
+  pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr);
+}
+
+/*
+** This method is the destructor fo a CsvTable object.
+*/
+static int csvtabDisconnect(sqlite3_vtab *pVtab){
+  CsvTable *p = (CsvTable*)pVtab;
+  sqlite3_free(p->zFilename);
+  sqlite3_free(p->zData);
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/* Skip leading whitespace.  Return a pointer to the first non-whitespace
+** character, or to the zero terminator if the string has only whitespace */
+static const char *csv_skip_whitespace(const char *z){
+  while( isspace((unsigned char)z[0]) ) z++;
+  return z;
+}
+
+/* Remove trailing whitespace from the end of string z[] */
+static void csv_trim_whitespace(char *z){
+  size_t n = strlen(z);
+  while( n>0 && isspace((unsigned char)z[n]) ) n--;
+  z[n] = 0;
+}
+
+/* Dequote the string */
+static void csv_dequote(char *z){
+  int i, j;
+  char cQuote = z[0];
+  size_t n;
+
+  if( cQuote!='\'' && cQuote!='"' ) return;
+  n = strlen(z);
+  if( n<2 || z[n-1]!=z[0] ) return;
+  for(i=1, j=0; i<n-1; i++){
+    if( z[i]==cQuote && z[i+1]==cQuote ) i++;
+    z[j++] = z[i];
+  }
+  z[j] = 0;
+}
+
+/* Check to see if the string is of the form:  "TAG = VALUE" with optional
+** whitespace before and around tokens.  If it is, return a pointer to the
+** first character of VALUE.  If it is not, return NULL.
+*/
+static const char *csv_parameter(const char *zTag, int nTag, const char *z){
+  z = csv_skip_whitespace(z);
+  if( strncmp(zTag, z, nTag)!=0 ) return 0;
+  z = csv_skip_whitespace(z+nTag);
+  if( z[0]!='=' ) return 0;
+  return csv_skip_whitespace(z+1);
+}
+
+/* Decode a parameter that requires a dequoted string.
+**
+** Return 1 if the parameter is seen, or 0 if not.  1 is returned
+** even if there is an error.  If an error occurs, then an error message
+** is left in p->zErr.  If there are no errors, p->zErr[0]==0.
+*/
+static int csv_string_parameter(
+  CsvReader *p,            /* Leave the error message here, if there is one */
+  const char *zParam,      /* Parameter we are checking for */
+  const char *zArg,        /* Raw text of the virtual table argment */
+  char **pzVal             /* Write the dequoted string value here */
+){
+  const char *zValue;
+  zValue = csv_parameter(zParam,strlen(zParam),zArg);
+  if( zValue==0 ) return 0;
+  p->zErr[0] = 0;
+  if( *pzVal ){
+    csv_errmsg(p, "more than one '%s' parameter", zParam);
+    return 1;
+  }
+  *pzVal = sqlite3_mprintf("%s", zValue);
+  if( *pzVal==0 ){
+    csv_errmsg(p, "out of memory");
+    return 1;
+  }
+  csv_trim_whitespace(*pzVal);
+  csv_dequote(*pzVal);
+  return 1;
+}
+
+
+/* Return 0 if the argument is false and 1 if it is true.  Return -1 if
+** we cannot really tell.
+*/
+static int csv_boolean(const char *z){
+  if( sqlite3_stricmp("yes",z)==0
+   || sqlite3_stricmp("on",z)==0
+   || sqlite3_stricmp("true",z)==0
+   || (z[0]=='1' && z[0]==0)
+  ){
+    return 1;
+  }
+  if( sqlite3_stricmp("no",z)==0
+   || sqlite3_stricmp("off",z)==0
+   || sqlite3_stricmp("false",z)==0
+   || (z[0]=='0' && z[1]==0)
+  ){
+    return 0;
+  }
+  return -1;
+}
+
+
+/*
+** Parameters:
+**    filename=FILENAME          Name of file containing CSV content
+**    data=TEXT                  Direct CSV content.
+**    schema=SCHEMA              Alternative CSV schema.
+**    header=YES|NO              First row of CSV defines the names of
+**                               columns if "yes".  Default "no".
+**    columns=N                  Assume the CSV file contains N columns.
+**
+** Only available if compiled with SQLITE_TEST:
+**    
+**    testflags=N                Bitmask of test flags.  Optional
+**
+** If schema= is omitted, then the columns are named "c0", "c1", "c2",
+** and so forth.  If columns=N is omitted, then the file is opened and
+** the number of columns in the first row is counted to determine the
+** column count.  If header=YES, then the first row is skipped.
+*/
+static int csvtabConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  CsvTable *pNew = 0;        /* The CsvTable object to construct */
+  int bHeader = -1;          /* header= flags.  -1 means not seen yet */
+  int rc = SQLITE_OK;        /* Result code from this routine */
+  int i, j;                  /* Loop counters */
+#ifdef SQLITE_TEST
+  int tstFlags = 0;          /* Value for testflags=N parameter */
+#endif
+  int nCol = -99;            /* Value of the columns= parameter */
+  CsvReader sRdr;            /* A CSV file reader used to store an error
+                             ** message and/or to count the number of columns */
+  static const char *azParam[] = {
+     "filename", "data", "schema", 
+  };
+  char *azPValue[3];         /* Parameter values */
+# define CSV_FILENAME (azPValue[0])
+# define CSV_DATA     (azPValue[1])
+# define CSV_SCHEMA   (azPValue[2])
+
+
+  assert( sizeof(azPValue)==sizeof(azParam) );
+  memset(&sRdr, 0, sizeof(sRdr));
+  memset(azPValue, 0, sizeof(azPValue));
+  for(i=3; i<argc; i++){
+    const char *z = argv[i];
+    const char *zValue;
+    for(j=0; j<sizeof(azParam)/sizeof(azParam[0]); j++){
+      if( csv_string_parameter(&sRdr, azParam[j], z, &azPValue[j]) ) break;
+    }
+    if( j<sizeof(azParam)/sizeof(azParam[0]) ){
+      if( sRdr.zErr[0] ) goto csvtab_connect_error;
+    }else
+    if( (zValue = csv_parameter("header",6,z))!=0 ){
+      int x;
+      if( bHeader>=0 ){
+        csv_errmsg(&sRdr, "more than one 'header' parameter");
+        goto csvtab_connect_error;
+      }
+      x = csv_boolean(zValue);
+      if( x==1 ){
+        bHeader = 1;
+      }else if( x==0 ){
+        bHeader = 0;
+      }else{
+        csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue);
+        goto csvtab_connect_error;
+      }
+    }else
+#ifdef SQLITE_TEST
+    if( (zValue = csv_parameter("testflags",9,z))!=0 ){
+      tstFlags = (unsigned int)atoi(zValue);
+    }else
+#endif
+    if( (zValue = csv_parameter("columns",7,z))!=0 ){
+      if( nCol>0 ){
+        csv_errmsg(&sRdr, "more than one 'columns' parameter");
+        goto csvtab_connect_error;
+      }
+      nCol = atoi(zValue);
+      if( nCol<=0 ){
+        csv_errmsg(&sRdr, "must have at least one column");
+        goto csvtab_connect_error;
+      }
+    }else
+    {
+      csv_errmsg(&sRdr, "unrecognized parameter '%s'", z);
+      goto csvtab_connect_error;
+    }
+  }
+  if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
+    csv_errmsg(&sRdr, "must either filename= or data= but not both");
+    goto csvtab_connect_error;
+  }
+  if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){
+    goto csvtab_connect_error;
+  }
+  pNew = sqlite3_malloc( sizeof(*pNew) );
+  *ppVtab = (sqlite3_vtab*)pNew;
+  if( pNew==0 ) goto csvtab_connect_oom;
+  memset(pNew, 0, sizeof(*pNew));
+  if( nCol>0 ){
+    pNew->nCol = nCol;
+  }else{
+    do{
+      const char *z = csv_read_one_field(&sRdr);
+      if( z==0 ) goto csvtab_connect_oom;
+      pNew->nCol++;
+    }while( sRdr.cTerm==',' );
+  }
+  pNew->zFilename = CSV_FILENAME;  CSV_FILENAME = 0;
+  pNew->zData = CSV_DATA;          CSV_DATA = 0;
+#ifdef SQLITE_TEST
+  pNew->tstFlags = tstFlags;
+#endif
+  pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0;
+  csv_reader_reset(&sRdr);
+  if( CSV_SCHEMA==0 ){
+    char *zSep = "";
+    CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x(");
+    if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
+    for(i=0; i<pNew->nCol; i++){
+      CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i);
+      zSep = ",";
+    }
+    CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA);
+  }
+  rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
+  if( rc ) goto csvtab_connect_error;
+  for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
+    sqlite3_free(azPValue[i]);
+  }
+  return SQLITE_OK;
+
+csvtab_connect_oom:
+  rc = SQLITE_NOMEM;
+  csv_errmsg(&sRdr, "out of memory");
+
+csvtab_connect_error:
+  if( pNew ) csvtabDisconnect(&pNew->base);
+  for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
+    sqlite3_free(azPValue[i]);
+  }
+  if( sRdr.zErr[0] ){
+    sqlite3_free(*pzErr);
+    *pzErr = sqlite3_mprintf("%s", sRdr.zErr);
+  }
+  csv_reader_reset(&sRdr);
+  if( rc==SQLITE_OK ) rc = SQLITE_ERROR;
+  return rc;
+}
+
+/*
+** Reset the current row content held by a CsvCursor.
+*/
+static void csvtabCursorRowReset(CsvCursor *pCur){
+  CsvTable *pTab = (CsvTable*)pCur->base.pVtab;
+  int i;
+  for(i=0; i<pTab->nCol; i++){
+    sqlite3_free(pCur->azVal[i]);
+    pCur->azVal[i] = 0;
+    pCur->aLen[i] = 0;
+  }
+}
+
+/*
+** The xConnect and xCreate methods do the same thing, but they must be
+** different so that the virtual table is not an eponymous virtual table.
+*/
+static int csvtabCreate(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+ return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr);
+}
+
+/*
+** Destructor for a CsvCursor.
+*/
+static int csvtabClose(sqlite3_vtab_cursor *cur){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  csvtabCursorRowReset(pCur);
+  csv_reader_reset(&pCur->rdr);
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new CsvTable cursor object.
+*/
+static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  CsvTable *pTab = (CsvTable*)p;
+  CsvCursor *pCur;
+  size_t nByte;
+  nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol;
+  pCur = sqlite3_malloc( nByte );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, nByte);
+  pCur->azVal = (char**)&pCur[1];
+  pCur->aLen = (int*)&pCur->azVal[pTab->nCol];
+  *ppCursor = &pCur->base;
+  if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){
+    csv_xfer_error(pTab, &pCur->rdr);
+    return SQLITE_ERROR;
+  }
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a CsvCursor to its next row of input.
+** Set the EOF marker if we reach the end of input.
+*/
+static int csvtabNext(sqlite3_vtab_cursor *cur){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  CsvTable *pTab = (CsvTable*)cur->pVtab;
+  int i = 0;
+  char *z;
+  do{
+    z = csv_read_one_field(&pCur->rdr);
+    if( z==0 ){
+      csv_xfer_error(pTab, &pCur->rdr);
+      break;
+    }
+    if( i<pTab->nCol ){
+      if( pCur->aLen[i] < pCur->rdr.n+1 ){
+        char *zNew = sqlite3_realloc(pCur->azVal[i], pCur->rdr.n+1);
+        if( zNew==0 ){
+          csv_errmsg(&pCur->rdr, "out of memory");
+          csv_xfer_error(pTab, &pCur->rdr);
+          break;
+        }
+        pCur->azVal[i] = zNew;
+        pCur->aLen[i] = pCur->rdr.n+1;
+      }
+      memcpy(pCur->azVal[i], z, pCur->rdr.n+1);
+      i++;
+    }
+  }while( pCur->rdr.cTerm==',' );
+  while( i<pTab->nCol ){
+    sqlite3_free(pCur->azVal[i]);
+    pCur->azVal[i] = 0;
+    pCur->aLen[i] = 0;
+    i++;
+  }
+  if( z==0 || pCur->rdr.cTerm==EOF ){
+    pCur->iRowid = -1;
+  }else{
+    pCur->iRowid++;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the CsvCursor
+** is currently pointing.
+*/
+static int csvtabColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  CsvTable *pTab = (CsvTable*)cur->pVtab;
+  if( i>=0 && i<pTab->nCol && pCur->azVal[i]!=0 ){
+    sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_STATIC);
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.
+*/
+static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int csvtabEof(sqlite3_vtab_cursor *cur){
+  CsvCursor *pCur = (CsvCursor*)cur;
+  return pCur->iRowid<0;
+}
+
+/*
+** Only a full table scan is supported.  So xFilter simply rewinds to
+** the beginning.
+*/
+static int csvtabFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  CsvCursor *pCur = (CsvCursor*)pVtabCursor;
+  CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab;
+  pCur->iRowid = 0;
+  if( pCur->rdr.in==0 ){
+    assert( pCur->rdr.zIn==pTab->zData );
+    assert( pTab->iStart<=pCur->rdr.nIn );
+    pCur->rdr.iIn = pTab->iStart;
+  }else{
+    fseek(pCur->rdr.in, pTab->iStart, SEEK_SET);
+    pCur->rdr.iIn = 0;
+    pCur->rdr.nIn = 0;
+  }
+  return csvtabNext(pVtabCursor);
+}
+
+/*
+** Only a forward full table scan is supported.  xBestIndex is mostly
+** a no-op.  If CSVTEST_FIDX is set, then the presence of equality
+** constraints lowers the estimated cost, which is fiction, but is useful
+** for testing certain kinds of virtual table behavior.
+*/
+static int csvtabBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = 1000000;
+#ifdef SQLITE_TEST
+  if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){
+    /* The usual (and sensible) case is to always do a full table scan.
+    ** The code in this branch only runs when testflags=1.  This code
+    ** generates an artifical and unrealistic plan which is useful
+    ** for testing virtual table logic but is not helpful to real applications.
+    **
+    ** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual
+    ** table (even though it is not) and the cost of running the virtual table
+    ** is reduced from 1 million to just 10.  The constraints are *not* marked
+    ** as omittable, however, so the query planner should still generate a
+    ** plan that gives a correct answer, even if they plan is not optimal.
+    */
+    int i;
+    int nConst = 0;
+    for(i=0; i<pIdxInfo->nConstraint; i++){
+      unsigned char op;
+      if( pIdxInfo->aConstraint[i].usable==0 ) continue;
+      op = pIdxInfo->aConstraint[i].op;
+      if( op==SQLITE_INDEX_CONSTRAINT_EQ 
+       || op==SQLITE_INDEX_CONSTRAINT_LIKE
+       || op==SQLITE_INDEX_CONSTRAINT_GLOB
+      ){
+        pIdxInfo->estimatedCost = 10;
+        pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1;
+        nConst++;
+      }
+    }
+  }
+#endif
+  return SQLITE_OK;
+}
+
+
+static sqlite3_module CsvModule = {
+  0,                       /* iVersion */
+  csvtabCreate,            /* xCreate */
+  csvtabConnect,           /* xConnect */
+  csvtabBestIndex,         /* xBestIndex */
+  csvtabDisconnect,        /* xDisconnect */
+  csvtabDisconnect,        /* xDestroy */
+  csvtabOpen,              /* xOpen - open a cursor */
+  csvtabClose,             /* xClose - close a cursor */
+  csvtabFilter,            /* xFilter - configure scan constraints */
+  csvtabNext,              /* xNext - advance a cursor */
+  csvtabEof,               /* xEof - check for end of scan */
+  csvtabColumn,            /* xColumn - read data */
+  csvtabRowid,             /* xRowid - read data */
+  0,                       /* xUpdate */
+  0,                       /* xBegin */
+  0,                       /* xSync */
+  0,                       /* xCommit */
+  0,                       /* xRollback */
+  0,                       /* xFindMethod */
+  0,                       /* xRename */
+};
+
+#ifdef SQLITE_TEST
+/*
+** For virtual table testing, make a version of the CSV virtual table
+** available that has an xUpdate function.  But the xUpdate always returns
+** SQLITE_READONLY since the CSV file is not really writable.
+*/
+static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){
+  return SQLITE_READONLY;
+}
+static sqlite3_module CsvModuleFauxWrite = {
+  0,                       /* iVersion */
+  csvtabCreate,            /* xCreate */
+  csvtabConnect,           /* xConnect */
+  csvtabBestIndex,         /* xBestIndex */
+  csvtabDisconnect,        /* xDisconnect */
+  csvtabDisconnect,        /* xDestroy */
+  csvtabOpen,              /* xOpen - open a cursor */
+  csvtabClose,             /* xClose - close a cursor */
+  csvtabFilter,            /* xFilter - configure scan constraints */
+  csvtabNext,              /* xNext - advance a cursor */
+  csvtabEof,               /* xEof - check for end of scan */
+  csvtabColumn,            /* xColumn - read data */
+  csvtabRowid,             /* xRowid - read data */
+  csvtabUpdate,            /* xUpdate */
+  0,                       /* xBegin */
+  0,                       /* xSync */
+  0,                       /* xCommit */
+  0,                       /* xRollback */
+  0,                       /* xFindMethod */
+  0,                       /* xRename */
+};
+#endif /* SQLITE_TEST */
+
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+/* 
+** This routine is called when the extension is loaded.  The new
+** CSV virtual table module is registered with the calling database
+** connection.
+*/
+int sqlite3_csv_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc;
+  SQLITE_EXTENSION_INIT2(pApi);
+  rc = sqlite3_create_module(db, "csv", &CsvModule, 0);
+#ifdef SQLITE_TEST
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0);
+  }
+#endif
+  return rc;
+}

ext/misc/vfsstat.c

@@ -0,0 +1,816 @@
+/*
+** 2016-05-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains the implementation of an SQLite vfs shim that
+** tracks I/O.  Access to the accumulated status counts is provided using
+** an eponymous virtual table.
+*/
+#include <sqlite3ext.h>
+SQLITE_EXTENSION_INIT1
+
+/*
+** This module contains code for a wrapper VFS that cause stats for
+** most VFS calls to be recorded.
+**
+** To use this module, first compile it as a loadable extension.  See
+** https://www.sqlite.org/loadext.html#build for compilations instructions.
+**
+** After compliing, load this extension, then open database connections to be
+** measured.  Query usages status using the vfsstat virtual table:
+**
+**         SELECT * FROM vfsstat;
+**
+** Reset counters using UPDATE statements against vfsstat:
+**
+**         UPDATE vfsstat SET count=0;
+**
+** EXAMPLE SCRIPT:
+**
+**      .load ./vfsstat
+**      .open test.db
+**      DROP TABLE IF EXISTS t1;
+**      CREATE TABLE t1(x,y);
+**      INSERT INTO t1 VALUES(123, randomblob(5000));
+**      CREATE INDEX t1x ON t1(x);
+**      DROP TABLE t1;
+**      VACUUM;
+**      SELECT * FROM vfsstat WHERE count>0;
+**
+** LIMITATIONS:
+** 
+** This module increments counters without using mutex protection.  So if
+** two or more threads try to use this module at the same time, race conditions
+** may occur which mess up the counts.  This is harmless, other than giving
+** incorrect statistics.
+*/
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+
+/*
+** File types
+*/
+#define VFSSTAT_MAIN         0   /* Main database file */
+#define VFSSTAT_JOURNAL      1   /* Rollback journal */
+#define VFSSTAT_WAL          2   /* Write-ahead log file */
+#define VFSSTAT_MASTERJRNL   3   /* Master journal */
+#define VFSSTAT_SUBJRNL      4   /* Subjournal */
+#define VFSSTAT_TEMPDB       5   /* TEMP database */
+#define VFSSTAT_TEMPJRNL     6   /* Journal for TEMP database */
+#define VFSSTAT_TRANSIENT    7   /* Transient database */
+#define VFSSTAT_ANY          8   /* Unspecified file type */
+#define VFSSTAT_nFile        9   /* This many file types */
+
+/* Names of the file types.  These are allowed values for the
+** first column of the vfsstat virtual table.
+*/
+static const char *azFile[] = {
+  "database", "journal", "wal", "master-journal", "sub-journal",
+  "temp-database", "temp-journal", "transient-db", "*"
+};
+
+/*
+** Stat types
+*/
+#define VFSSTAT_BYTESIN      0   /* Bytes read in */
+#define VFSSTAT_BYTESOUT     1   /* Bytes written out */   
+#define VFSSTAT_READ         2   /* Read requests */
+#define VFSSTAT_WRITE        3   /* Write requests */
+#define VFSSTAT_SYNC         4   /* Syncs */
+#define VFSSTAT_OPEN         5   /* File opens */
+#define VFSSTAT_LOCK         6   /* Lock requests */
+#define VFSSTAT_ACCESS       0   /* xAccess calls.  filetype==ANY only */
+#define VFSSTAT_DELETE       1   /* xDelete calls.  filetype==ANY only */
+#define VFSSTAT_FULLPATH     2   /* xFullPathname calls.  ANY only */
+#define VFSSTAT_RANDOM       3   /* xRandomness calls.    ANY only */
+#define VFSSTAT_SLEEP        4   /* xSleep calls.         ANY only */
+#define VFSSTAT_CURTIME      5   /* xCurrentTime calls.   ANY only */
+#define VFSSTAT_nStat        7   /* This many stat types */
+
+
+/* Names for the second column of the vfsstat virtual table for all
+** cases except when the first column is "*" or VFSSTAT_ANY. */
+static const char *azStat[] = {
+  "bytes-in", "bytes-out", "read", "write", "sync", "open", "lock",
+};
+static const char *azStatAny[] = {
+  "access", "delete", "fullpathname", "randomness", "sleep", "currenttimestamp",
+  "not-used"
+};
+
+/* Total number of counters */
+#define VFSSTAT_MXCNT  (VFSSTAT_nStat*VFSSTAT_nFile)
+
+/*
+** Performance stats are collected in an instance of the following
+** global array.
+*/
+static sqlite3_uint64 aVfsCnt[VFSSTAT_MXCNT];
+
+/*
+** Access to a specific counter
+*/
+#define STATCNT(filetype,stat) (aVfsCnt[(filetype)*VFSSTAT_nStat+(stat)])
+
+/*
+** Forward declaration of objects used by this utility
+*/
+typedef struct VStatVfs VStatVfs;
+typedef struct VStatFile VStatFile;
+
+/* An instance of the VFS */
+struct VStatVfs {
+  sqlite3_vfs base;               /* VFS methods */
+  sqlite3_vfs *pVfs;              /* Parent VFS */
+};
+
+/* An open file */
+struct VStatFile {
+  sqlite3_file base;              /* IO methods */
+  sqlite3_file *pReal;            /* Underlying file handle */
+  unsigned char eFiletype;        /* What type of file is this */
+};
+
+#define REALVFS(p) (((VStatVfs*)(p))->pVfs)
+
+/*
+** Methods for VStatFile
+*/
+static int vstatClose(sqlite3_file*);
+static int vstatRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int vstatWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
+static int vstatTruncate(sqlite3_file*, sqlite3_int64 size);
+static int vstatSync(sqlite3_file*, int flags);
+static int vstatFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int vstatLock(sqlite3_file*, int);
+static int vstatUnlock(sqlite3_file*, int);
+static int vstatCheckReservedLock(sqlite3_file*, int *pResOut);
+static int vstatFileControl(sqlite3_file*, int op, void *pArg);
+static int vstatSectorSize(sqlite3_file*);
+static int vstatDeviceCharacteristics(sqlite3_file*);
+static int vstatShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
+static int vstatShmLock(sqlite3_file*, int offset, int n, int flags);
+static void vstatShmBarrier(sqlite3_file*);
+static int vstatShmUnmap(sqlite3_file*, int deleteFlag);
+static int vstatFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
+static int vstatUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
+
+/*
+** Methods for VStatVfs
+*/
+static int vstatOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
+static int vstatDelete(sqlite3_vfs*, const char *zName, int syncDir);
+static int vstatAccess(sqlite3_vfs*, const char *zName, int flags, int *);
+static int vstatFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
+static void *vstatDlOpen(sqlite3_vfs*, const char *zFilename);
+static void vstatDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
+static void (*vstatDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
+static void vstatDlClose(sqlite3_vfs*, void*);
+static int vstatRandomness(sqlite3_vfs*, int nByte, char *zOut);
+static int vstatSleep(sqlite3_vfs*, int microseconds);
+static int vstatCurrentTime(sqlite3_vfs*, double*);
+static int vstatGetLastError(sqlite3_vfs*, int, char *);
+static int vstatCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+
+static VStatVfs vstat_vfs = {
+  {
+    2,                            /* iVersion */
+    0,                            /* szOsFile (set by register_vstat()) */
+    1024,                         /* mxPathname */
+    0,                            /* pNext */
+    "vfslog",                     /* zName */
+    0,                            /* pAppData */
+    vstatOpen,                     /* xOpen */
+    vstatDelete,                   /* xDelete */
+    vstatAccess,                   /* xAccess */
+    vstatFullPathname,             /* xFullPathname */
+    vstatDlOpen,                   /* xDlOpen */
+    vstatDlError,                  /* xDlError */
+    vstatDlSym,                    /* xDlSym */
+    vstatDlClose,                  /* xDlClose */
+    vstatRandomness,               /* xRandomness */
+    vstatSleep,                    /* xSleep */
+    vstatCurrentTime,              /* xCurrentTime */
+    vstatGetLastError,             /* xGetLastError */
+    vstatCurrentTimeInt64          /* xCurrentTimeInt64 */
+  },
+  0
+};
+
+static const sqlite3_io_methods vstat_io_methods = {
+  3,                              /* iVersion */
+  vstatClose,                      /* xClose */
+  vstatRead,                       /* xRead */
+  vstatWrite,                      /* xWrite */
+  vstatTruncate,                   /* xTruncate */
+  vstatSync,                       /* xSync */
+  vstatFileSize,                   /* xFileSize */
+  vstatLock,                       /* xLock */
+  vstatUnlock,                     /* xUnlock */
+  vstatCheckReservedLock,          /* xCheckReservedLock */
+  vstatFileControl,                /* xFileControl */
+  vstatSectorSize,                 /* xSectorSize */
+  vstatDeviceCharacteristics,      /* xDeviceCharacteristics */
+  vstatShmMap,                     /* xShmMap */
+  vstatShmLock,                    /* xShmLock */
+  vstatShmBarrier,                 /* xShmBarrier */
+  vstatShmUnmap,                   /* xShmUnmap */
+  vstatFetch,                      /* xFetch */
+  vstatUnfetch                     /* xUnfetch */
+};
+
+
+
+/*
+** Close an vstat-file.
+*/
+static int vstatClose(sqlite3_file *pFile){
+  VStatFile *p = (VStatFile *)pFile;
+  int rc = SQLITE_OK;
+
+  if( p->pReal->pMethods ){
+    rc = p->pReal->pMethods->xClose(p->pReal);
+  }
+  return rc;
+}
+
+
+/*
+** Read data from an vstat-file.
+*/
+static int vstatRead(
+  sqlite3_file *pFile, 
+  void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+
+  rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
+  STATCNT(p->eFiletype,VFSSTAT_READ)++;
+  if( rc==SQLITE_OK ){
+    STATCNT(p->eFiletype,VFSSTAT_BYTESIN) += iAmt;
+  }
+  return rc;
+}
+
+/*
+** Write data to an vstat-file.
+*/
+static int vstatWrite(
+  sqlite3_file *pFile,
+  const void *z,
+  int iAmt,
+  sqlite_int64 iOfst
+){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+
+  rc = p->pReal->pMethods->xWrite(p->pReal, z, iAmt, iOfst);
+  STATCNT(p->eFiletype,VFSSTAT_WRITE)++;
+  if( rc==SQLITE_OK ){
+    STATCNT(p->eFiletype,VFSSTAT_BYTESOUT) += iAmt;
+  }
+  return rc;
+}
+
+/*
+** Truncate an vstat-file.
+*/
+static int vstatTruncate(sqlite3_file *pFile, sqlite_int64 size){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xTruncate(p->pReal, size);
+  return rc;
+}
+
+/*
+** Sync an vstat-file.
+*/
+static int vstatSync(sqlite3_file *pFile, int flags){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xSync(p->pReal, flags);
+  STATCNT(p->eFiletype,VFSSTAT_SYNC)++;
+  return rc;
+}
+
+/*
+** Return the current file-size of an vstat-file.
+*/
+static int vstatFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
+  return rc;
+}
+
+/*
+** Lock an vstat-file.
+*/
+static int vstatLock(sqlite3_file *pFile, int eLock){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xLock(p->pReal, eLock);
+  STATCNT(p->eFiletype,VFSSTAT_LOCK)++;
+  return rc;
+}
+
+/*
+** Unlock an vstat-file.
+*/
+static int vstatUnlock(sqlite3_file *pFile, int eLock){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xUnlock(p->pReal, eLock);
+  STATCNT(p->eFiletype,VFSSTAT_LOCK)++;
+  return rc;
+}
+
+/*
+** Check if another file-handle holds a RESERVED lock on an vstat-file.
+*/
+static int vstatCheckReservedLock(sqlite3_file *pFile, int *pResOut){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut);
+  STATCNT(p->eFiletype,VFSSTAT_LOCK)++;
+  return rc;
+}
+
+/*
+** File control method. For custom operations on an vstat-file.
+*/
+static int vstatFileControl(sqlite3_file *pFile, int op, void *pArg){
+  VStatFile *p = (VStatFile *)pFile;
+  int rc;
+  rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg);
+  if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){
+    *(char**)pArg = sqlite3_mprintf("vstat/%z", *(char**)pArg);
+  }
+  return rc;
+}
+
+/*
+** Return the sector-size in bytes for an vstat-file.
+*/
+static int vstatSectorSize(sqlite3_file *pFile){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xSectorSize(p->pReal);
+  return rc;
+}
+
+/*
+** Return the device characteristic flags supported by an vstat-file.
+*/
+static int vstatDeviceCharacteristics(sqlite3_file *pFile){
+  int rc;
+  VStatFile *p = (VStatFile *)pFile;
+  rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal);
+  return rc;
+}
+
+/* Create a shared memory file mapping */
+static int vstatShmMap(
+  sqlite3_file *pFile,
+  int iPg,
+  int pgsz,
+  int bExtend,
+  void volatile **pp
+){
+  VStatFile *p = (VStatFile *)pFile;
+  return p->pReal->pMethods->xShmMap(p->pReal, iPg, pgsz, bExtend, pp);
+}
+
+/* Perform locking on a shared-memory segment */
+static int vstatShmLock(sqlite3_file *pFile, int offset, int n, int flags){
+  VStatFile *p = (VStatFile *)pFile;
+  return p->pReal->pMethods->xShmLock(p->pReal, offset, n, flags);
+}
+
+/* Memory barrier operation on shared memory */
+static void vstatShmBarrier(sqlite3_file *pFile){
+  VStatFile *p = (VStatFile *)pFile;
+  p->pReal->pMethods->xShmBarrier(p->pReal);
+}
+
+/* Unmap a shared memory segment */
+static int vstatShmUnmap(sqlite3_file *pFile, int deleteFlag){
+  VStatFile *p = (VStatFile *)pFile;
+  return p->pReal->pMethods->xShmUnmap(p->pReal, deleteFlag);
+}
+
+/* Fetch a page of a memory-mapped file */
+static int vstatFetch(
+  sqlite3_file *pFile,
+  sqlite3_int64 iOfst,
+  int iAmt,
+  void **pp
+){
+  VStatFile *p = (VStatFile *)pFile;
+  return p->pReal->pMethods->xFetch(p->pReal, iOfst, iAmt, pp);
+}
+
+/* Release a memory-mapped page */
+static int vstatUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
+  VStatFile *p = (VStatFile *)pFile;
+  return p->pReal->pMethods->xUnfetch(p->pReal, iOfst, pPage);
+}
+
+/*
+** Open an vstat file handle.
+*/
+static int vstatOpen(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_file *pFile,
+  int flags,
+  int *pOutFlags
+){
+  int rc;
+  VStatFile *p = (VStatFile*)pFile;
+
+  p->pReal = (sqlite3_file*)&p[1];
+  rc = REALVFS(pVfs)->xOpen(REALVFS(pVfs), zName, p->pReal, flags, pOutFlags);
+  if( flags & SQLITE_OPEN_MAIN_DB ){
+    p->eFiletype = VFSSTAT_MAIN;
+  }else if( flags & SQLITE_OPEN_MAIN_JOURNAL ){
+    p->eFiletype = VFSSTAT_JOURNAL;
+  }else if( flags & SQLITE_OPEN_WAL ){
+    p->eFiletype = VFSSTAT_WAL;
+  }else if( flags & SQLITE_OPEN_MASTER_JOURNAL ){
+    p->eFiletype = VFSSTAT_MASTERJRNL;
+  }else if( flags & SQLITE_OPEN_SUBJOURNAL ){
+    p->eFiletype = VFSSTAT_SUBJRNL;
+  }else if( flags & SQLITE_OPEN_TEMP_DB ){
+    p->eFiletype = VFSSTAT_TEMPDB;
+  }else if( flags & SQLITE_OPEN_TEMP_JOURNAL ){
+    p->eFiletype = VFSSTAT_TEMPJRNL;
+  }else{
+    p->eFiletype = VFSSTAT_TRANSIENT;
+  }
+  STATCNT(p->eFiletype,VFSSTAT_OPEN)++;
+  pFile->pMethods = rc ? 0 : &vstat_io_methods;
+  return rc;
+}
+
+/*
+** Delete the file located at zPath. If the dirSync argument is true,
+** ensure the file-system modifications are synced to disk before
+** returning.
+*/
+static int vstatDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
+  int rc;
+  rc = REALVFS(pVfs)->xDelete(REALVFS(pVfs), zPath, dirSync);
+  STATCNT(VFSSTAT_ANY,VFSSTAT_DELETE)++;
+  return rc;
+}
+
+/*
+** Test for access permissions. Return true if the requested permission
+** is available, or false otherwise.
+*/
+static int vstatAccess(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int flags, 
+  int *pResOut
+){
+  int rc;
+  rc = REALVFS(pVfs)->xAccess(REALVFS(pVfs), zPath, flags, pResOut);
+  STATCNT(VFSSTAT_ANY,VFSSTAT_ACCESS)++;
+  return rc;
+}
+
+/*
+** Populate buffer zOut with the full canonical pathname corresponding
+** to the pathname in zPath. zOut is guaranteed to point to a buffer
+** of at least (INST_MAX_PATHNAME+1) bytes.
+*/
+static int vstatFullPathname(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int nOut, 
+  char *zOut
+){
+  STATCNT(VFSSTAT_ANY,VFSSTAT_FULLPATH)++;
+  return REALVFS(pVfs)->xFullPathname(REALVFS(pVfs), zPath, nOut, zOut);
+}
+
+/*
+** Open the dynamic library located at zPath and return a handle.
+*/
+static void *vstatDlOpen(sqlite3_vfs *pVfs, const char *zPath){
+  return REALVFS(pVfs)->xDlOpen(REALVFS(pVfs), zPath);
+}
+
+/*
+** Populate the buffer zErrMsg (size nByte bytes) with a human readable
+** utf-8 string describing the most recent error encountered associated 
+** with dynamic libraries.
+*/
+static void vstatDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
+  REALVFS(pVfs)->xDlError(REALVFS(pVfs), nByte, zErrMsg);
+}
+
+/*
+** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
+*/
+static void (*vstatDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
+  return REALVFS(pVfs)->xDlSym(REALVFS(pVfs), p, zSym);
+}
+
+/*
+** Close the dynamic library handle pHandle.
+*/
+static void vstatDlClose(sqlite3_vfs *pVfs, void *pHandle){
+  REALVFS(pVfs)->xDlClose(REALVFS(pVfs), pHandle);
+}
+
+/*
+** Populate the buffer pointed to by zBufOut with nByte bytes of 
+** random data.
+*/
+static int vstatRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+  STATCNT(VFSSTAT_ANY,VFSSTAT_RANDOM)++;
+  return REALVFS(pVfs)->xRandomness(REALVFS(pVfs), nByte, zBufOut);
+}
+
+/*
+** Sleep for nMicro microseconds. Return the number of microseconds 
+** actually slept.
+*/
+static int vstatSleep(sqlite3_vfs *pVfs, int nMicro){
+  STATCNT(VFSSTAT_ANY,VFSSTAT_SLEEP)++;
+  return REALVFS(pVfs)->xSleep(REALVFS(pVfs), nMicro);
+}
+
+/*
+** Return the current time as a Julian Day number in *pTimeOut.
+*/
+static int vstatCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
+  STATCNT(VFSSTAT_ANY,VFSSTAT_CURTIME)++;
+  return REALVFS(pVfs)->xCurrentTime(REALVFS(pVfs), pTimeOut);
+}
+
+static int vstatGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+  return REALVFS(pVfs)->xGetLastError(REALVFS(pVfs), a, b);
+}
+static int vstatCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
+  STATCNT(VFSSTAT_ANY,VFSSTAT_CURTIME)++;
+  return REALVFS(pVfs)->xCurrentTimeInt64(REALVFS(pVfs), p);
+}
+
+/*
+** A virtual table for accessing the stats collected by this VFS shim
+*/
+static int vstattabConnect(sqlite3*, void*, int, const char*const*, 
+                           sqlite3_vtab**,char**);
+static int vstattabBestIndex(sqlite3_vtab*,sqlite3_index_info*);
+static int vstattabDisconnect(sqlite3_vtab*);
+static int vstattabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**);
+static int vstattabClose(sqlite3_vtab_cursor*);
+static int vstattabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr,
+                          int argc, sqlite3_value **argv);
+static int vstattabNext(sqlite3_vtab_cursor*);
+static int vstattabEof(sqlite3_vtab_cursor*);
+static int vstattabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int);
+static int vstattabRowid(sqlite3_vtab_cursor*,sqlite3_int64*);
+static int vstattabUpdate(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
+
+/* A cursor for the vfsstat virtual table */
+typedef struct VfsStatCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  int i;                          /* Pointing to this aVfsCnt[] value */
+} VfsStatCursor;
+
+
+static int vstattabConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  sqlite3_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define VSTAT_COLUMN_FILE  0 
+#define VSTAT_COLUMN_STAT  1
+#define VSTAT_COLUMN_COUNT 2
+
+  rc = sqlite3_declare_vtab(db,"CREATE TABLE x(file,stat,count)");
+  if( rc==SQLITE_OK ){
+    pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) );
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for vstat table object.
+*/
+static int vstattabDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new vstat table cursor object.
+*/
+static int vstattabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  VfsStatCursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+
+/*
+** Destructor for a VfsStatCursor.
+*/
+static int vstattabClose(sqlite3_vtab_cursor *cur){
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a VfsStatCursor to its next row of output.
+*/
+static int vstattabNext(sqlite3_vtab_cursor *cur){
+  ((VfsStatCursor*)cur)->i++;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the VfsStatCursor
+** is currently pointing.
+*/
+static int vstattabColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  VfsStatCursor *pCur = (VfsStatCursor*)cur;
+  switch( i ){
+    case VSTAT_COLUMN_FILE: {
+      sqlite3_result_text(ctx, azFile[pCur->i/VFSSTAT_nStat], -1, SQLITE_STATIC);
+      break;
+    }
+    case VSTAT_COLUMN_STAT: {
+      const char **az;
+      az = (pCur->i/VFSSTAT_nStat)==VFSSTAT_ANY ? azStatAny : azStat;
+      sqlite3_result_text(ctx, az[pCur->i%VFSSTAT_nStat], -1, SQLITE_STATIC);
+      break;
+    }
+    case VSTAT_COLUMN_COUNT: {
+      sqlite3_result_int64(ctx, aVfsCnt[pCur->i]);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.
+*/
+static int vstattabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  VfsStatCursor *pCur = (VfsStatCursor*)cur;
+  *pRowid = pCur->i;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int vstattabEof(sqlite3_vtab_cursor *cur){
+  VfsStatCursor *pCur = (VfsStatCursor*)cur;
+  return pCur->i >= VFSSTAT_MXCNT;
+}
+
+/*
+** Only a full table scan is supported.  So xFilter simply rewinds to
+** the beginning.
+*/
+static int vstattabFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  VfsStatCursor *pCur = (VfsStatCursor*)pVtabCursor;
+  pCur->i = 0;
+  return SQLITE_OK;
+}
+
+/*
+** Only a forwards full table scan is supported.  xBestIndex is a no-op.
+*/
+static int vstattabBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  return SQLITE_OK;
+}
+
+/*
+** Any VSTAT_COLUMN_COUNT can be changed to a positive integer.
+** No deletions or insertions are allowed.  No changes to other
+** columns are allowed.
+*/
+static int vstattabUpdate(
+  sqlite3_vtab *tab,
+  int argc, sqlite3_value **argv,
+  sqlite3_int64 *pRowid
+){
+  sqlite3_int64 iRowid, x;
+  if( argc==1 ) return SQLITE_ERROR;
+  if( sqlite3_value_type(argv[0])!=SQLITE_INTEGER ) return SQLITE_ERROR;
+  iRowid = sqlite3_value_int64(argv[0]);
+  if( iRowid!=sqlite3_value_int64(argv[1]) ) return SQLITE_ERROR;
+  if( iRowid<0 || iRowid>=VFSSTAT_MXCNT ) return SQLITE_ERROR;
+  if( sqlite3_value_type(argv[VSTAT_COLUMN_COUNT+2])!=SQLITE_INTEGER ){
+    return SQLITE_ERROR;
+  }
+  x = sqlite3_value_int64(argv[VSTAT_COLUMN_COUNT+2]);
+  if( x<0 ) return SQLITE_ERROR;
+  aVfsCnt[iRowid] = x;
+  return SQLITE_OK;
+}
+
+static sqlite3_module VfsStatModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  vstattabConnect,           /* xConnect */
+  vstattabBestIndex,         /* xBestIndex */
+  vstattabDisconnect,        /* xDisconnect */
+  0,                         /* xDestroy */
+  vstattabOpen,              /* xOpen - open a cursor */
+  vstattabClose,             /* xClose - close a cursor */
+  vstattabFilter,            /* xFilter - configure scan constraints */
+  vstattabNext,              /* xNext - advance a cursor */
+  vstattabEof,               /* xEof - check for end of scan */
+  vstattabColumn,            /* xColumn - read data */
+  vstattabRowid,             /* xRowid - read data */
+  vstattabUpdate,            /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+};
+
+/*
+** This routine is an sqlite3_auto_extension() callback, invoked to register
+** the vfsstat virtual table for all new database connections.
+*/
+static int vstatRegister(
+  sqlite3 *db,
+  const char **pzErrMsg,
+  const struct sqlite3_api_routines *pThunk
+){
+  return sqlite3_create_module(db, "vfsstat", &VfsStatModule, 0);
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+/* 
+** This routine is called when the extension is loaded.
+**
+** Register the new VFS.  Make arrangement to register the virtual table
+** for each new database connection.
+*/
+int sqlite3_vfsstat_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  vstat_vfs.pVfs = sqlite3_vfs_find(0);
+  vstat_vfs.base.szOsFile = sizeof(VStatFile) + vstat_vfs.pVfs->szOsFile;
+  rc = sqlite3_vfs_register(&vstat_vfs.base, 1);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_auto_extension((void(*)(void))vstatRegister);
+  }
+  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
+  return rc;
+}

ext/rbu/rbu_common.tcl

@@ -36,3 +36,20 @@ proc step_rbu {target rbu} {
   set rc
 }
 
+proc do_rbu_vacuum_test {tn step} {
+  uplevel [list do_test $tn.1 {
+    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
+    while 1 {
+      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
+      set rc [rbu step]
+      if {$rc!="SQLITE_OK"} break
+      if {$step==1} { rbu close }
+    }
+    rbu close
+  } {SQLITE_DONE}]
+
+  uplevel [list do_execsql_test $tn.2 {
+    PRAGMA integrity_check
+  } ok]
+}
+

ext/rbu/rbudiff.test

@@ -25,6 +25,10 @@ proc get_rbudiff_sql {db1 db2} {
   exec $::PROG --rbu $db1 $db2
 }
 
+proc get_vtab_rbudiff_sql {db1 db2} {
+  exec $::PROG --vtab --rbu $db1 $db2
+}
+
 proc step_rbu {target rbu} {
   while 1 {
     sqlite3rbu rbu $target $rbu
@@ -44,6 +48,11 @@ proc apply_rbudiff {sql target} {
   step_rbu $target rbu.db
 }
 
+proc sqlesc {id} {
+  set ret "'[string map {' ''} $id]'"
+  set ret
+}
+
 # The only argument is the output of an [sqldiff -rbu] run. This command
 # tests that the contents of the rbu_count table is correct. An exception
 # is thrown if it is not.
@@ -54,7 +63,7 @@ proc test_rbucount {sql} {
   tmpdb eval {
     SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table'
   } {
-    set a [tmpdb eval "SELECT count(*) FROM $name"]
+    set a [tmpdb eval "SELECT count(*) FROM [sqlesc $name]"]
     set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}]
     if {$a != $b} { 
       tmpdb close
@@ -71,9 +80,11 @@ proc rbudiff_cksum {db1} {
   sqlite3 dbtmp $db1
   foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] {
     set cols [list]
-    dbtmp eval "PRAGMA table_info = $tbl" { lappend cols "quote( $name )" }
+    dbtmp eval "PRAGMA table_info = [sqlesc $tbl]" { 
+      lappend cols "quote( $name )" 
+    }
     append txt [dbtmp eval \
-      "SELECT [join $cols {||'.'||}] FROM $tbl ORDER BY 1"
+      "SELECT [join $cols {||'.'||}] FROM [sqlesc $tbl] ORDER BY 1"
     ]
   }
   dbtmp close
@@ -159,5 +170,57 @@ foreach {tn init mod} {
   do_test 1.$tn.5 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
 }
 
+#-------------------------------------------------------------------------
+# Test that if the --vtab switch is present, [sqldiff] handles virtual
+# table types fts[345] and rtree correctly.
+#
+ifcapable fts3&&fts5&&rtree {
+
+foreach {tn init mod} {
+  1 {
+    CREATE VIRTUAL TABLE t1 USING fts5(c);
+    INSERT INTO t1 VALUES('a b c');
+    INSERT INTO t1 VALUES('a b c');
+  } {
+    DELETE FROM t1 WHERE rowid = 1;
+    INSERT INTO t1 VALUES('a b c');
+  }
+
+  2 {
+    CREATE VIRTUAL TABLE "x y" USING 'rtree'(id, x1, x2);
+    INSERT INTO "x y" VALUES(1, 2, 3);
+    INSERT INTO "x y" VALUES(2, 4, 6);
+  } {
+    DELETE FROM "x y" WHERE rowid = 1;
+    INSERT INTO "x y" VALUES(3, 6, 9);
+  }
+
+  3 {
+    CREATE VIRTUAL TABLE 'x''y' USING fts3;
+    INSERT INTO 'x''y' VALUES('one two three');
+    INSERT INTO 'x''y' VALUES('four five six');
+  } {
+    DELETE FROM 'x''y' WHERE rowid = 1;
+    INSERT INTO 'x''y' VALUES('one two three');
+  }
+
+} {
+
+  forcedelete test.db test.db2
+  sqlite3 db test.db
+  db eval "$init"
+  sqlite3 db test.db2
+  db eval "$init ; $mod"
+  db close
+
+  do_test 2.$tn.1 {
+    set sql [get_vtab_rbudiff_sql test.db test.db2]
+    apply_rbudiff $sql test.db
+  } {SQLITE_DONE}
+  do_test 2.$tn.2 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
+}
+
+}
+
 finish_test
 

ext/rbu/rbuvacuum.test

@@ -17,23 +17,6 @@
 source [file join [file dirname [info script]] rbu_common.tcl]
 set ::testprefix rbuvacuum
 
-proc do_rbu_vacuum_test {tn step} {
-  uplevel [list do_test $tn.1 {
-    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
-    while 1 {
-      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
-      set rc [rbu step]
-      if {$rc!="SQLITE_OK"} break
-      if {$step==1} { rbu close }
-    }
-    rbu close
-  } {SQLITE_DONE}]
-
-  uplevel [list do_execsql_test $tn.2 {
-    PRAGMA integrity_check
-  } ok]
-}
-
 foreach step {0 1} {
 
   set ::testprefix rbuvacuum-step=$step
@@ -404,7 +387,6 @@ do_test 3.5 {
   list [catch { rbu close } msg] $msg
 } {0 SQLITE_DONE}
 
-
 catch { db close }
 finish_test
 

ext/rbu/rbuvacuum2.test

@@ -0,0 +1,162 @@
+# 2016 June 1
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the RBU module. More specifically, it
+# contains tests to ensure that the sqlite3rbu_vacuum() API works as
+# expected.
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+
+foreach step {0 1} {
+  set ::testprefix rbuvacuum2-$step
+  
+  #-------------------------------------------------------------------------
+  # Test that a database that contains fts3 tables can be vacuumed.
+  #
+  ifcapable fts3 {
+    reset_db
+    do_execsql_test 1.1 {
+      CREATE VIRTUAL TABLE t1 USING fts3(z, y);
+      INSERT INTO t1 VALUES('fix this issue', 'at some point');
+    }
+  
+    do_rbu_vacuum_test 1.2 $step
+  
+    do_execsql_test 1.3 {
+      SELECT * FROM t1;
+    } {{fix this issue} {at some point}}
+  
+    do_execsql_test 1.4 {
+      SELECT rowid FROM t1 WHERE t1 MATCH 'fix';
+    } {1}
+
+    do_execsql_test 1.5 {
+      INSERT INTO t1 VALUES('a b c', 'd e f');
+      INSERT INTO t1 VALUES('l h i', 'd e f');
+      DELETE FROM t1 WHERE docid = 2;
+      INSERT INTO t1 VALUES('a b c', 'x y z');
+    }
+
+    do_rbu_vacuum_test 1.6 $step
+    do_execsql_test 1.7 {
+      INSERT INTO t1(t1) VALUES('integrity-check');
+      SELECT * FROM t1;
+    } {
+      {fix this issue} {at some point}
+      {l h i} {d e f}
+      {a b c} {x y z}
+    }
+  }
+  
+  #-------------------------------------------------------------------------
+  # Test that a database that contains fts5 tables can be vacuumed.
+  #
+  ifcapable fts5 {
+    reset_db
+    do_execsql_test 2.1 {
+      CREATE VIRTUAL TABLE t1 USING fts5(z, y);
+      INSERT INTO t1 VALUES('fix this issue', 'at some point');
+    }
+  
+    do_rbu_vacuum_test 2.2 $step
+  
+    do_execsql_test 2.3 {
+      SELECT * FROM t1;
+    } {{fix this issue} {at some point}}
+  
+    do_execsql_test 2.4 {
+      SELECT rowid FROM t1 ('fix');
+    } {1}
+
+    do_execsql_test 2.5 {
+      INSERT INTO t1 VALUES('a b c', 'd e f');
+      INSERT INTO t1 VALUES('l h i', 'd e f');
+      DELETE FROM t1 WHERE rowid = 2;
+      INSERT INTO t1 VALUES('a b c', 'x y z');
+    }
+
+    do_rbu_vacuum_test 2.6 $step
+    do_execsql_test 2.7 {
+      INSERT INTO t1(t1) VALUES('integrity-check');
+      SELECT * FROM t1;
+    } {
+      {fix this issue} {at some point}
+      {l h i} {d e f}
+      {a b c} {x y z}
+    }
+  }
+
+  #-------------------------------------------------------------------------
+  # Test that a database that contains an rtree table can be vacuumed.
+  #
+  ifcapable rtree {
+    reset_db
+    do_execsql_test 3.1 {
+      CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2);
+      INSERT INTO rt VALUES(1, 45, 55);
+      INSERT INTO rt VALUES(2, 50, 60);
+      INSERT INTO rt VALUES(3, 55, 65);
+    }
+  
+    do_rbu_vacuum_test 3.2 $step
+  
+    do_execsql_test 3.3 {
+      SELECT * FROM rt;
+    } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0}
+  
+    do_execsql_test 3.4.1 {
+      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
+    } {1 2}
+    do_execsql_test 3.4.2 {
+      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
+    } {2 3}
+
+    do_rbu_vacuum_test 3.5 $step
+
+    do_execsql_test 3.6.1 {
+      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
+    } {1 2}
+    do_execsql_test 3.6.2 {
+      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
+    } {2 3}
+  }
+
+  ifcapable trigger {
+    reset_db
+    do_execsql_test 4.1 {
+      CREATE TABLE t1(a, b, c);
+      INSERT INTO t1 VALUES(1, 2, 3);
+      CREATE VIEW v1 AS SELECT * FROM t1;
+      CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END;
+    }
+
+    do_execsql_test 4.2 {
+      SELECT * FROM sqlite_master;
+    } {
+    table t1 t1 2 {CREATE TABLE t1(a, b, c)}
+    view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
+    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
+    }
+
+    do_rbu_vacuum_test 4.3 $step
+    do_execsql_test 4.4 {
+      SELECT * FROM sqlite_master;
+    } {
+    table t1 t1 2 {CREATE TABLE t1(a, b, c)}
+    view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
+    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
+    }
+  }
+
+}
+
+finish_test

ext/rbu/sqlite3rbu.c

@@ -921,12 +921,14 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){
   int rc;
   memset(pIter, 0, sizeof(RbuObjIter));
 
-  rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, 
+  rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, 
+    sqlite3_mprintf(
       "SELECT rbu_target_name(name, type='view') AS target, name "
       "FROM sqlite_master "
       "WHERE type IN ('table', 'view') AND target IS NOT NULL "
+      " %s "
       "ORDER BY name"
-  );
+  , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : ""));
 
   if( rc==SQLITE_OK ){
     rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg,
@@ -2505,9 +2507,9 @@ static void rbuFileSuffix3(const char *zBase, char *z){
 #endif
   {
     int i, sz;
-    sz = sqlite3Strlen30(z);
+    sz = (int)strlen(z)&0xffffff;
     for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
-    if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4);
+    if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4);
   }
 #endif
 }

main.mk

@@ -324,7 +324,9 @@ TESTSRC = \
 #
 TESTSRC += \
   $(TOP)/ext/misc/amatch.c \
+  $(TOP)/ext/misc/carray.c \
   $(TOP)/ext/misc/closure.c \
+  $(TOP)/ext/misc/csv.c \
   $(TOP)/ext/misc/eval.c \
   $(TOP)/ext/misc/fileio.c \
   $(TOP)/ext/misc/fuzzer.c \
@@ -450,7 +452,8 @@ TESTPROGS = \
   testfixture$(EXE) \
   sqlite3$(EXE) \
   sqlite3_analyzer$(EXE) \
-  sqldiff$(EXE)
+  sqldiff$(EXE) \
+  dbhash$(EXE)
 
 # Databases containing fuzzer test cases
 #
@@ -487,6 +490,10 @@ sqldiff$(EXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
 	$(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \
 		$(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB)
 
+dbhash$(EXE):	$(TOP)/tool/dbhash.c sqlite3.c sqlite3.h
+	$(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \
+		$(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB)
+
 scrub$(EXE):	$(TOP)/ext/misc/scrub.c sqlite3.o
 	$(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB)
 

manifest

@@ -1,8 +1,8 @@
-C Clang\scan\sdefine\s_MSC_VER\sin\ssome\scircumstances;\stherefore,\scheck\sfor\sClang\sfirst.
-D 2016-05-28T00:13:21.036
-F Makefile.in f59e0763ff448719fc1bd25513882b0567286317
+C Change\sthe\sname\sof\sthe\sintarray()\sextension\sto\scarray()\sand\sgive\sit\san\noptional\sthird\sparameter\sthat\sspecifies\sthe\sdatatype\sas\sone\sof\s'int32',\n'int64',\s'double',\sor\s'char*'.\s\s'int32'\sis\sthe\sdefault.
+D 2016-07-03T02:35:47.798
+F Makefile.in 6c20d44f72d4564f11652b26291a214c8367e5db
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 306d73e854b1a92ea06e5d1e637faa5c44de53c7
+F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
 F VERSION cb29eb11e493dd85b3eeec4053c03949bf98478e
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -14,7 +14,7 @@ F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4
 F autoconf/Makefile.msc 0eca137c12542bd76c253a2d24380f29ade59b95
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
-F autoconf/configure.ac 7754404e955900cfdeba08862cdfa802eca5b0af
+F autoconf/configure.ac cacf2616abf6e4a569bde2ef365c143caeec40bc
 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd
 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873
 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43
@@ -98,14 +98,14 @@ F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h b770c5e0a8d2ee071ddffc7ab722dbf3474a8abe
-F ext/fts5/fts5Int.h 3677076aecbf645a7f2a019115c6a4ec3272dd78
+F ext/fts5/fts5Int.h 9bd0c7c64285b5b368eca0ac63613185c5ad24ba
 F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
-F ext/fts5/fts5_expr.c da2b33c2aac91e96641c0a7cf2bbaa36eb2667f7
+F ext/fts5/fts5_expr.c bcb238ee4ac1164302ab528487520488516bd030
 F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2
 F ext/fts5/fts5_index.c b429e23fabb57506f71e406997cc46b89190dc97
-F ext/fts5/fts5_main.c b4a0fc5bf17f2f1f056ee76cdd7d2af08b360f55
+F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2
 F ext/fts5/fts5_storage.c 3309c6a8e34b974513016fd1ef47c83f5898f94c
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
 F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc
@@ -176,7 +176,7 @@ F ext/fts5/test/fts5porter.test 7cdc07bef301d70eebbfa75dcaf45c3680e1d0e1
 F ext/fts5/test/fts5porter2.test 2e65633d58a1c525d5af0f6c01e5a59155bb3487
 F ext/fts5/test/fts5prefix.test efd42e00bb8e8a36383f25c838185508681c093f
 F ext/fts5/test/fts5query.test f5ec25f5f2fbb70033424113cdffc101b1985a40
-F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab
+F ext/fts5/test/fts5rank.test 3e55e7eeb4c98728e4a3171c9e994e1a2f24eb99
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6
@@ -204,8 +204,10 @@ F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c 43df9d8ef2fae7a325100ebd713ab089dc829dd7
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
 F ext/misc/amatch.c 211108e201105e4bb0c076527b8cfd34330fc234
+F ext/misc/carray.c f1947c7d5bbce17f8244c9a05baae11d5d68467e w ext/misc/array.c
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
+F ext/misc/csv.c f01126ba170fd4ef7c752b156568a80c912d4441
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
@@ -221,6 +223,7 @@ F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c bf1b922c2750698e9a3d4c50cce6974adb7e93be
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
+F ext/misc/vfsstat.c 6110aeeaab2f1df17a923c8a8acef3c74f6dc515
 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
 F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4
@@ -239,17 +242,18 @@ F ext/rbu/rbu9.test 0806d1772c9f4981774ff028de6656e4183082af
 F ext/rbu/rbuA.test c1a7b3e2d926b8f8448bb3b4ae787e314ee4b2b3
 F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2
 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831
-F ext/rbu/rbu_common.tcl 0398545fed614f807d5f0ba55a85a51f08ba8f1a
+F ext/rbu/rbu_common.tcl 3a4b916b6f5dca9c9da9a30863e272fe5ea4414f
 F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695
-F ext/rbu/rbudiff.test 2df0a8a7d998ecf81764c21eeda3cde5611c5091
+F ext/rbu/rbudiff.test 4c9f8df6f723f553781d3d117501b7e9d170a145
 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89
 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca
 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda
 F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a
 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
-F ext/rbu/rbuvacuum.test 66e02cf299836770e718e95c36686be0b26dbda3
-F ext/rbu/sqlite3rbu.c bf36625990c6865ecf08bd844d8097ed2d0a6958
+F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0
+F ext/rbu/rbuvacuum2.test 45009e127c3fb385e5c0fd5a8a63fb922a79d0ab
+F ext/rbu/sqlite3rbu.c 3d35b1dc46f5ca0f0f59db8e6163fe7627ea107c
 F ext/rbu/sqlite3rbu.h 2acd0a6344a6079de15c8dc9d84d3df83a665930
 F ext/rbu/test_rbu.c 9bbdf6bd8efd58fbc4f192698df50569598fbb9e
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
@@ -304,7 +308,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk eab7db27ac2a32b1172a85aa850c0b9c3a00524c
+F main.mk e9d66f1b1c4874221d12d940be3ce5f397c10741
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -323,27 +327,27 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
 F src/backup.c 6df65fdd569c901a418887a1a76f82ec35044556
 F src/bitvec.c 3ee4c8b2c94ed3a7377256e18199e6ff5cf33f63
 F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73
-F src/btree.c 2128172fc1c420a6fa6878827fa595407795069a
-F src/btree.h 1342a9b2cc2089e3534d3ef00204786783f6aea6
+F src/btree.c 0621247619df15790f7ed5f14926cfb3925c10b1
+F src/btree.h 2107a2630e02c8cba58bb12ce14e731e734ea29c
 F src/btreeInt.h c18b7d2a3494695133e4e60ee36061d37f45d9a5
-F src/build.c 785fa789319d93c6ae20efbd01d4da9ce8f8a793
+F src/build.c d1fdfd7ab8f5447e494ef15825973bf0719527c6
 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
-F src/ctime.c 999a828425b35b8092a8cde25690e71c20906344
+F src/ctime.c 61949e83c4c36e37195a8398ebc752780b534d95
 F src/date.c 1cc9fb516ec9932c6fd4d2a0d2f8bc4480145c39
-F src/dbstat.c c845548d4346e606e2f2b7d2e714ace2b8a7dd1b
+F src/dbstat.c 4f6f7f52b49beb9636ffbd517cfe44a402ba4ad0
 F src/delete.c 4aba4214a377ce8ddde2d2e609777bcc8235200f
-F src/expr.c 81bd7d87985746313770211183e900ed1ad28381
+F src/expr.c 523a5b1db2b6d88c6eefb224877bf635a3bcfc92
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
-F src/fkey.c 4c0bd09e602b8ae8d36d81e31e4872d0b53c87bb
-F src/func.c ef4c18c8a66143413ce41a58d582d2c14ddf78e1
+F src/fkey.c bc4145347595b7770f9a598cff1c848302cf5413
+F src/func.c 61a4114cf7004f10c542cfabbab9f2bcb9033045
 F src/global.c c45ea22aff29334f6a9ec549235ac3357c970015
 F src/hash.c 55b5fb474100cee0b901edaf203e26c970940f36
 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
 F src/insert.c 8f4e9fcbd8e95e85f15647ba8b413b18d556ec2b
 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e
-F src/loadext.c 8b3a73f0624c5f7cadbd5cb89940783bee1d39a6
+F src/loadext.c 4237fd37ca589f1d90b3ea925dd9df20da8e0c84
 F src/main.c 405d13e3a4f7c5add9fb27702ae70ed0a6e32cca
 F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -364,34 +368,34 @@ F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
 F src/os_unix.c a9443cdab41d7f3cdf0df3a5aab62fd6e1c9b234
-F src/os_win.c 852fc2ff6084296348ed3739c548b2cf32df394e
+F src/os_win.c d4b8faf8896b65818e67070711fdd00d8e620bd6
 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
 F src/pager.c c368634b888b1c8740aea83b36bfd266f2443e60
 F src/pager.h 8ab6b6feeee4bc0439bfde7ee59ba99df98b9bc3
-F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e
+F src/parse.y f374ab20106362eb3f5c01b3e6a002f0bbead7ff
 F src/pcache.c 5583c8ade4b05075a60ba953ef471d1c1a9c05df
 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490
 F src/pcache1.c 7f51d2b541aab57596adf62db2c4bb025d34f04d
-F src/pragma.c d98039af2deb5a4990f6635462b3f61f325a6c75
+F src/pragma.c c8b499756658cb8b82cfdbb5845c22cf11f297aa
 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c
 F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e
 F src/printf.c a5f0ca08ddede803c241266abb46356ec748ded1
 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696
 F src/resolve.c cca3aa77b95706df5d635a2141a4d1de60ae6598
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c a0c4abf54bc6bd3a9c77a36ef3d1676045706cb2
+F src/select.c f3c6e9065fb34f6a23af27ec7f1f717ffbfc2ee4
 F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6
-F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54
+F src/sqlite.h.in 5f8113dbec74c6c093ead9930afb8c9fbd9f643d
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 98f72cbfe00169c39089115427d06ea05fe4b4a2
-F src/sqliteInt.h 09621b4b7dba808b24262c2480ea75b045001853
+F src/sqliteInt.h 2ac73e9fdd138f4607139d9bd6a1e6dcd01837dc
 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247
 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9
-F src/tclsqlite.c 9c4c4589d078de37813ded708d8838b338ffb060
-F src/test1.c c0e5b69f99e95a2c9f55fdb6e46b44f1a15292d8
+F src/tclsqlite.c 25fbbbb97f76dbfd113153fb63f52d7ecfac5dd0
+F src/test1.c 7cbfda741fbfa541ecce305ff2d7c6bb730aa991
 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b
-F src/test3.c d2c9efd2985ff8f5502ffd3253156984778d77d8
+F src/test3.c c75c8af0eadb335236c9e61b51044c58a8f7dd59
 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e
 F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1
 F src/test6.c a684b7abd01352ab50cb79c0bf727e6b3f381a3d
@@ -445,24 +449,24 @@ F src/update.c 4f05ea8cddfa367d045e03589756c02199e8f9bd
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c 810ec3f22e2d1b62e66c30fe3621ebdedd23584d
 F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52
-F src/vdbe.c 45e4da739187028281e342f79fae0f4145055bec
+F src/vdbe.c 22b46c3b725e950e9f2760e2d76953d592600ad4
 F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170
 F src/vdbeInt.h ddb157974436d87652de7dc641f7191496d9a8cd
-F src/vdbeapi.c ba85b78fe08dc4a9ce747e62c89a2b4a4547e74c
-F src/vdbeaux.c 1d6b9a979d1036db7bc39990e9e683f19520bc5c
-F src/vdbeblob.c c9f2f494b911c6fa34efd9803f0a10807da80f77
-F src/vdbemem.c 5cfef60e60e19cab6275d1b975bf4c791d575beb
+F src/vdbeapi.c 02bcbc2ca5d2004b029088b05b468b394881e103
+F src/vdbeaux.c c90275b0e55a2b32c03dc09314194fe46f2429d8
+F src/vdbeblob.c 83d2d266383157b02e2b809350bb197e89d7895b
+F src/vdbemem.c 1ecaa5ee0caff07255f25d04e8dc88befb6f88d1
 F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c
 F src/vdbetrace.c f75c5455d8cf389ef86a8bfdfd3177e0e3692484
-F src/vtab.c ce0f2ebb589b459b32c640b33af64bfa5b29aaf8
+F src/vtab.c 948d2d4984219eee37a7bf427d6667e21e6eb92e
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 02eeecc265f6ffd0597378f5d8ae9070b62a406a
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c b9f5b0ddb14c3827e70b5379e659cf4cfd524c4d
+F src/where.c 48eed8ebe319c6cbc7bf7682018f32af0f5189f5
 F src/whereInt.h e5b939701a7ceffc5a3a8188a37f9746416ebcd0
-F src/wherecode.c ba71a4e4bada29aa9842200e6299714bf18c812c
-F src/whereexpr.c eacc0e60d029a082b4fc0cc42ea98544add1319e
+F src/wherecode.c e20cb381ff621e56a4684c71e31999aca2547ca6
+F src/whereexpr.c d7dcbf14ce1b5876c1f76496162c30fcba669563
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -525,6 +529,7 @@ F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f
 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c
 F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee
 F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060
+F test/bestindex3.test b80da904d23581d233a7ceee7d6bbad2b23a8133
 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c
 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59
 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc
@@ -570,6 +575,7 @@ F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868
 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334
 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a
 F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
+F test/collateB.test 8ec2accd2d7166c1eff0d2a39bc90262c6f89632
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b
 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
@@ -610,8 +616,10 @@ F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d
 F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2
 F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8
 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c
-F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47
+F test/csv01.test 0929a9ce47021519512be92861f29e32d2538e5f
+F test/ctime.test 0b995accd44a52914bd4744d5c1b6e1a56c7897c
 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856
+F test/cursorhint2.test 2b45341d32d1aae9991a00ef31ebca339b274255
 F test/date.test 984ac1e3e5e031386866f034006148d3972b4a65
 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204
 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab
@@ -835,7 +843,7 @@ F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407
 F test/index3.test 81bc47890b8abfb181bc35f8d10b56c069803386
 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6
 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7
-F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a
+F test/index6.test 43b4e29258b978fcdab84fc61df4f5212119dd09
 F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c
 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985
 F test/indexexpr1.test cb71b6586177b840e28110dd952178bb2bdfedc2
@@ -1035,7 +1043,7 @@ F test/select6.test 39eac4a5c03650b2b473c532882273283ee8b7a0
 F test/select7.test 95e370c42d47c3c52377d05e9ffc01ccff7c1f61
 F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d
 F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95
-F test/selectA.test e452bdb975f488ea46d091382a9185b5853ed2c7
+F test/selectA.test 101e722370ac6e84978c2958b8931c78b10a1709
 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25
 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977
 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394
@@ -1088,14 +1096,14 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/speedtest1.c 4476f7030775507f060d27b789e1f4123c5a5866
+F test/speedtest1.c 28221f433d358dd3fcf2ca504ed51409aa66ffaf
 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e
 F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142
 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a
-F test/stat.test 66e95f97b9f724f9ab921d054ee0db3c2689f1ee
+F test/stat.test ab95d28503d0f6d98ffd8ce204643c9da090ebf1
 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1
 F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f
 F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f
@@ -1108,11 +1116,11 @@ F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c
 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04
-F test/tabfunc01.test f977868fa8bb7beb4b2072883190411653473906
+F test/tabfunc01.test 50a9fb379f9747fd0d40ea6d8fa3a101361bb537
 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f
 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126
 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930
-F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa
+F test/tclsqlite.test cf0d0a3fd03d64892cec2d48aae9fb2f148680a5
 F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6
 F test/tempdb2.test 4fc92055f2a3f7626c0f2eabbb637dc021b311d5
 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900
@@ -1356,7 +1364,7 @@ F test/walcksum.test bb234a1bb42248b3515d992b719708015c384278
 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a
 F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36
 F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af
-F test/walcrash4.test fa7d92ab84fe3ff409e1cda88a622545e2bc2b66
+F test/walcrash4.test 3374d6a0813bfe9d841a973958e0552b545a6423
 F test/walfault.test 1f8389f7709877e9b4cc679033d71d6fe529056b
 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483
 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c
@@ -1370,7 +1378,7 @@ F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
 F test/where.test f0c325563acde44f2c4ea6ba348e9e29f7121757
 F test/where2.test 478d2170637b9211f593120648858593bf2445a1
-F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e
+F test/where3.test 54cdeb02157acc979de41530b804ae7b09552bf1
 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8
 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b
@@ -1415,6 +1423,7 @@ F tool/build-all-msvc.bat 3e4e4043b53f1aede4308e0d2567bbd773614630 x
 F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
 F tool/cg_anno.tcl 692ce4b8693d59e3a3de77ca97f4139ecfa641b0 x
 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2
+F tool/dbhash.c a06228aa21ebc4e6ea8daa486601d938499238a5
 F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2
 F tool/fast_vacuum.c 5ba0d6f5963a0a63bdc42840f678bad75b2ebce1
 F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439
@@ -1423,7 +1432,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/lemon.c 09a96bed19955697a5e20c49ad863ec2005815a2
-F tool/lempar.c f06b7e98a6b7efb404375b6f4a0c71c85faa1512
+F tool/lempar.c 8c4e9d8517e50da391f1d89a519e743dd4afbc09
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1462,7 +1471,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
-F tool/sqldiff.c ca315aca4e2d24233e8f2000edea5880c53d1875
+F tool/sqldiff.c 131e9c6e09d3d78e3c4ae8b5fd23451521723aac
 F tool/srcck1.c 4c39bdfa9a92edd20233ee720df84dbeb2417602
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1495,7 +1504,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7c2cd4b05f88a759bdf72d3a249db5984fd7105d
-R 49f2ff6c03dd42bcf8b96859a75667c2
-U mistachkin
-Z 7fd0da57a78dfad092dddeef636c580c
+P 7c3d441f2a9f642f3d91dcee854a4d16d298bc34
+R 41f36cbba6be0bbd66a667562505a546
+U drh
+Z 31f2105436c0dcfd40b6904e7b94f7b1

manifest.uuid

@@ -1 +1 @@
-3f710bc3617691fa7432a24f1410d4651ca181b6
+a204ba99db34b356acb259189158a32d2df25da0

src/btree.c

@@ -450,6 +450,15 @@ static void releasePage(MemPage *pPage);  /* Forward reference */
 static int cursorHoldsMutex(BtCursor *p){
   return sqlite3_mutex_held(p->pBt->mutex);
 }
+
+/* Verify that the cursor and the BtShared agree about what is the current
+** database connetion. This is important in shared-cache mode. If the database 
+** connection pointers get out-of-sync, it is possible for routines like
+** btreeInitPage() to reference an stale connection pointer that references a
+** a connection that has already closed.  This routine is used inside assert()
+** statements only and for the purpose of double-checking that the btree code
+** does keep the database connection pointers up-to-date.
+*/
 static int cursorOwnsBtShared(BtCursor *p){
   assert( cursorHoldsMutex(p) );
   return (p->pBtree->db==p->pBt->db);
@@ -609,21 +618,19 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){
 ** the key.
 */
 static int saveCursorKey(BtCursor *pCur){
-  int rc;
+  int rc = SQLITE_OK;
   assert( CURSOR_VALID==pCur->eState );
   assert( 0==pCur->pKey );
   assert( cursorHoldsMutex(pCur) );
 
-  rc = sqlite3BtreeKeySize(pCur, &pCur->nKey);
-  assert( rc==SQLITE_OK );  /* KeySize() cannot fail */
-
-  /* If this is an intKey table, then the above call to BtreeKeySize()
-  ** stores the integer key in pCur->nKey. In this case this value is
-  ** all that is required. Otherwise, if pCur is not open on an intKey
-  ** table, then malloc space for and store the pCur->nKey bytes of key 
-  ** data.  */
-  if( 0==pCur->curIntKey ){
-    void *pKey = sqlite3Malloc( pCur->nKey );
+  if( pCur->curIntKey ){
+    /* Only the rowid is required for a table btree */
+    pCur->nKey = sqlite3BtreeIntegerKey(pCur);
+  }else{
+    /* For an index btree, save the complete key content */
+    void *pKey;
+    pCur->nKey = sqlite3BtreePayloadSize(pCur);
+    pKey = sqlite3Malloc( pCur->nKey );
     if( pKey ){
       rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey);
       if( rc==SQLITE_OK ){
@@ -4262,46 +4269,33 @@ int sqlite3BtreeCursorIsValid(BtCursor *pCur){
 #endif /* NDEBUG */
 
 /*
-** Set *pSize to the size of the buffer needed to hold the value of
-** the key for the current entry.  If the cursor is not pointing
-** to a valid entry, *pSize is set to 0. 
-**
-** For a table with the INTKEY flag set, this routine returns the key
-** itself, not the number of bytes in the key.
-**
-** The caller must position the cursor prior to invoking this routine.
-** 
-** This routine cannot fail.  It always returns SQLITE_OK.  
+** Return the value of the integer key or "rowid" for a table btree.
+** This routine is only valid for a cursor that is pointing into a
+** ordinary table btree.  If the cursor points to an index btree or
+** is invalid, the result of this routine is undefined.
 */
-int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){
+i64 sqlite3BtreeIntegerKey(BtCursor *pCur){
   assert( cursorHoldsMutex(pCur) );
   assert( pCur->eState==CURSOR_VALID );
+  assert( pCur->curIntKey );
   getCellInfo(pCur);
-  *pSize = pCur->info.nKey;
-  return SQLITE_OK;
+  return pCur->info.nKey;
 }
 
 /*
-** Set *pSize to the number of bytes of data in the entry the
-** cursor currently points to.
+** Return the number of bytes of payload for the entry that pCur is
+** currently pointing to.  For table btrees, this will be the amount
+** of data.  For index btrees, this will be the size of the key.
 **
 ** The caller must guarantee that the cursor is pointing to a non-NULL
 ** valid entry.  In other words, the calling procedure must guarantee
 ** that the cursor has Cursor.eState==CURSOR_VALID.
-**
-** Failure is not possible.  This function always returns SQLITE_OK.
-** It might just as well be a procedure (returning void) but we continue
-** to return an integer result code for historical reasons.
 */
-int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){
-  assert( cursorOwnsBtShared(pCur) );
+u32 sqlite3BtreePayloadSize(BtCursor *pCur){
+  assert( cursorHoldsMutex(pCur) );
   assert( pCur->eState==CURSOR_VALID );
-  assert( pCur->iPage>=0 );
-  assert( pCur->iPage<BTCURSOR_MAX_DEPTH );
-  assert( pCur->apPage[pCur->iPage]->intKeyLeaf==1 );
   getCellInfo(pCur);
-  *pSize = pCur->info.nPayload;
-  return SQLITE_OK;
+  return pCur->info.nPayload;
 }
 
 /*
@@ -4743,10 +4737,7 @@ static const void *fetchPayload(
 ** These routines is used to get quick access to key and data
 ** in the common case where no overflow pages are used.
 */
-const void *sqlite3BtreeKeyFetch(BtCursor *pCur, u32 *pAmt){
-  return fetchPayload(pCur, pAmt);
-}
-const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){
+const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){
   return fetchPayload(pCur, pAmt);
 }
 
@@ -5079,11 +5070,12 @@ int sqlite3BtreeMovetoUnpacked(
   assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
   assert( pRes );
   assert( (pIdxKey==0)==(pCur->pKeyInfo==0) );
+  assert( pCur->eState!=CURSOR_VALID || (pIdxKey==0)==(pCur->curIntKey!=0) );
 
   /* If the cursor is already positioned at the point we are trying
   ** to move to, then just return without doing any work */
-  if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0
-   && pCur->curIntKey 
+  if( pIdxKey==0
+   && pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0
   ){
     if( pCur->info.nKey==intKey ){
       *pRes = 0;

src/btree.h

@@ -284,11 +284,10 @@ int sqlite3BtreeLast(BtCursor*, int *pRes);
 int sqlite3BtreeNext(BtCursor*, int *pRes);
 int sqlite3BtreeEof(BtCursor*);
 int sqlite3BtreePrevious(BtCursor*, int *pRes);
-int sqlite3BtreeKeySize(BtCursor*, i64 *pSize);
+i64 sqlite3BtreeIntegerKey(BtCursor*);
 int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*);
-const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt);
-const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt);
-int sqlite3BtreeDataSize(BtCursor*, u32 *pSize);
+const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt);
+u32 sqlite3BtreePayloadSize(BtCursor*);
 int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*);
 
 char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*);

src/build.c

@@ -338,7 +338,7 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){
 */
 Table *sqlite3LocateTable(
   Parse *pParse,         /* context in which to report errors */
-  int isView,            /* True if looking for a VIEW rather than a TABLE */
+  u32 flags,             /* LOCATE_VIEW or LOCATE_NOERR */
   const char *zName,     /* Name of the table we are looking for */
   const char *zDbase     /* Name of the database.  Might be NULL */
 ){
@@ -352,7 +352,7 @@ Table *sqlite3LocateTable(
 
   p = sqlite3FindTable(pParse->db, zName, zDbase);
   if( p==0 ){
-    const char *zMsg = isView ? "no such view" : "no such table";
+    const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table";
 #ifndef SQLITE_OMIT_VIRTUALTABLE
     if( sqlite3FindDbName(pParse->db, zDbase)<1 ){
       /* If zName is the not the name of a table in the schema created using
@@ -364,12 +364,14 @@ Table *sqlite3LocateTable(
       }
     }
 #endif
-    if( zDbase ){
-      sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName);
-    }else{
-      sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName);
+    if( (flags & LOCATE_NOERR)==0 ){
+      if( zDbase ){
+        sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName);
+      }else{
+        sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName);
+      }
+      pParse->checkSchema = 1;
     }
-    pParse->checkSchema = 1;
   }
 
   return p;
@@ -386,7 +388,7 @@ Table *sqlite3LocateTable(
 */
 Table *sqlite3LocateTableItem(
   Parse *pParse, 
-  int isView, 
+  u32 flags,
   struct SrcList_item *p
 ){
   const char *zDb;
@@ -397,7 +399,7 @@ Table *sqlite3LocateTableItem(
   }else{
     zDb = p->zDatabase;
   }
-  return sqlite3LocateTable(pParse, isView, p->zName, zDb);
+  return sqlite3LocateTable(pParse, flags, p->zName, zDb);
 }
 
 /*
@@ -605,8 +607,9 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
   /* Delete all indices associated with this table. */
   for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){
     pNext = pIndex->pNext;
-    assert( pIndex->pSchema==pTable->pSchema );
-    if( !db || db->pnBytesFreed==0 ){
+    assert( pIndex->pSchema==pTable->pSchema
+         || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) );
+    if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){
       char *zName = pIndex->zName; 
       TESTONLY ( Index *pOld = ) sqlite3HashInsert(
          &pIndex->pSchema->idxHash, zName, 0
@@ -1288,7 +1291,7 @@ void sqlite3AddPrimaryKey(
   Column *pCol = 0;
   int iCol = -1, i;
   int nTerm;
-  if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit;
+  if( pTab==0 ) goto primary_key_exit;
   if( pTab->tabFlags & TF_HasPrimaryKey ){
     sqlite3ErrorMsg(pParse, 
       "table \"%s\" has more than one primary key", pTab->zName);
@@ -1334,12 +1337,8 @@ void sqlite3AddPrimaryKey(
        "INTEGER PRIMARY KEY");
 #endif
   }else{
-    Index *p;
-    p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0,
-                           0, sortOrder, 0);
-    if( p ){
-      p->idxType = SQLITE_IDXTYPE_PRIMARYKEY;
-    }
+    sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0,
+                           0, sortOrder, 0, SQLITE_IDXTYPE_PRIMARYKEY);
     pList = 0;
   }
 
@@ -1656,21 +1655,23 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){
 ** are appropriate for a WITHOUT ROWID table instead of a rowid table.
 ** Changes include:
 **
-**     (1)  Convert the OP_CreateTable into an OP_CreateIndex.  There is
+**     (1)  Set all columns of the PRIMARY KEY schema object to be NOT NULL.
+**     (2)  Convert the OP_CreateTable into an OP_CreateIndex.  There is
 **          no rowid btree for a WITHOUT ROWID.  Instead, the canonical
 **          data storage is a covering index btree.
-**     (2)  Bypass the creation of the sqlite_master table entry
+**     (3)  Bypass the creation of the sqlite_master table entry
 **          for the PRIMARY KEY as the primary key index is now
 **          identified by the sqlite_master table entry of the table itself.
-**     (3)  Set the Index.tnum of the PRIMARY KEY Index object in the
+**     (4)  Set the Index.tnum of the PRIMARY KEY Index object in the
 **          schema to the rootpage from the main table.
-**     (4)  Set all columns of the PRIMARY KEY schema object to be NOT NULL.
 **     (5)  Add all table columns to the PRIMARY KEY Index object
 **          so that the PRIMARY KEY is a covering index.  The surplus
 **          columns are part of KeyInfo.nXField and are not used for
 **          sorting or lookup or uniqueness checks.
 **     (6)  Replace the rowid tail on all automatically generated UNIQUE
 **          indices with the PRIMARY KEY columns.
+**
+** For virtual tables, only (1) is performed.
 */
 static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
   Index *pIdx;
@@ -1680,6 +1681,20 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
   sqlite3 *db = pParse->db;
   Vdbe *v = pParse->pVdbe;
 
+  /* Mark every PRIMARY KEY column as NOT NULL (except for imposter tables)
+  */
+  if( !db->init.imposterTable ){
+    for(i=0; i<pTab->nCol; i++){
+      if( (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 ){
+        pTab->aCol[i].notNull = OE_Abort;
+      }
+    }
+  }
+
+  /* The remaining transformations only apply to b-tree tables, not to
+  ** virtual tables */
+  if( IN_DECLARE_VTAB ) return;
+
   /* Convert the OP_CreateTable opcode that would normally create the
   ** root-page for the table into an OP_CreateIndex opcode.  The index
   ** created will become the PRIMARY KEY index.
@@ -1701,9 +1716,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
     if( pList==0 ) return;
     pList->a[0].sortOrder = pParse->iPkSortOrder;
     assert( pParse->pNewTable==pTab );
-    pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0);
-    if( pPk==0 ) return;
-    pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY;
+    sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0,
+                       SQLITE_IDXTYPE_PRIMARYKEY);
+    if( db->mallocFailed ) return;
+    pPk = sqlite3PrimaryKeyIndex(pTab);
     pTab->iPKey = -1;
   }else{
     pPk = sqlite3PrimaryKeyIndex(pTab);
@@ -1731,19 +1747,11 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
     }
     pPk->nKeyCol = j;
   }
-  pPk->isCovering = 1;
   assert( pPk!=0 );
+  pPk->isCovering = 1;
+  if( !db->init.imposterTable ) pPk->uniqNotNull = 1;
   nPk = pPk->nKeyCol;
 
-  /* Make sure every column of the PRIMARY KEY is NOT NULL.  (Except,
-  ** do not enforce this for imposter tables.) */
-  if( !db->init.imposterTable ){
-    for(i=0; i<nPk; i++){
-      pTab->aCol[pPk->aiColumn[i]].notNull = OE_Abort;
-    }
-    pPk->uniqNotNull = 1;
-  }
-
   /* The root page of the PRIMARY KEY is the table root page */
   pPk->tnum = pTab->tnum;
 
@@ -2498,6 +2506,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){
   assert( pName->nSrc==1 );
   if( sqlite3ReadSchema(pParse) ) goto exit_drop_table;
   if( noErr ) db->suppressErr++;
+  assert( isView==0 || isView==LOCATE_VIEW );
   pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]);
   if( noErr ) db->suppressErr--;
 
@@ -2868,12 +2877,8 @@ Index *sqlite3AllocateIndexObject(
 ** pList is a list of columns to be indexed.  pList will be NULL if this
 ** is a primary key or unique-constraint on the most recent column added
 ** to the table currently under construction.  
-**
-** If the index is created successfully, return a pointer to the new Index
-** structure. This is used by sqlite3AddPrimaryKey() to mark the index
-** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY)
 */
-Index *sqlite3CreateIndex(
+void sqlite3CreateIndex(
   Parse *pParse,     /* All information about this parse */
   Token *pName1,     /* First part of index name. May be NULL */
   Token *pName2,     /* Second part of index name. May be NULL */
@@ -2883,9 +2888,9 @@ Index *sqlite3CreateIndex(
   Token *pStart,     /* The CREATE token that begins this statement */
   Expr *pPIWhere,    /* WHERE clause for partial indices */
   int sortOrder,     /* Sort order of primary key when pList==NULL */
-  int ifNotExist     /* Omit error if index already exists */
+  int ifNotExist,    /* Omit error if index already exists */
+  u8 idxType         /* The index type */
 ){
-  Index *pRet = 0;     /* Pointer to return */
   Table *pTab = 0;     /* Table to be indexed */
   Index *pIndex = 0;   /* The index to be created */
   char *zName = 0;     /* Name of the index */
@@ -2903,7 +2908,10 @@ Index *sqlite3CreateIndex(
   char *zExtra = 0;                /* Extra space after the Index object */
   Index *pPk = 0;      /* PRIMARY KEY index for WITHOUT ROWID tables */
 
-  if( db->mallocFailed || IN_DECLARE_VTAB || pParse->nErr>0 ){
+  if( db->mallocFailed || pParse->nErr>0 ){
+    goto exit_create_index;
+  }
+  if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){
     goto exit_create_index;
   }
   if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
@@ -3092,7 +3100,7 @@ Index *sqlite3CreateIndex(
   pIndex->pTable = pTab;
   pIndex->onError = (u8)onError;
   pIndex->uniqNotNull = onError!=OE_None;
-  pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE;
+  pIndex->idxType = idxType;
   pIndex->pSchema = db->aDb[iDb].pSchema;
   pIndex->nKeyCol = pList->nExpr;
   if( pPIWhere ){
@@ -3272,7 +3280,7 @@ Index *sqlite3CreateIndex(
             pIdx->onError = pIndex->onError;
           }
         }
-        pRet = pIdx;
+        if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType;
         goto exit_create_index;
       }
     }
@@ -3284,6 +3292,7 @@ Index *sqlite3CreateIndex(
   assert( pParse->nErr==0 );
   if( db->init.busy ){
     Index *p;
+    assert( !IN_DECLARE_VTAB );
     assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) );
     p = sqlite3HashInsert(&pIndex->pSchema->idxHash, 
                           pIndex->zName, pIndex);
@@ -3390,7 +3399,6 @@ Index *sqlite3CreateIndex(
       pIndex->pNext = pOther->pNext;
       pOther->pNext = pIndex;
     }
-    pRet = pIndex;
     pIndex = 0;
   }
 
@@ -3401,7 +3409,6 @@ exit_create_index:
   sqlite3ExprListDelete(db, pList);
   sqlite3SrcListDelete(db, pTblName);
   sqlite3DbFree(db, zName);
-  return pRet;
 }
 
 /*
@@ -3430,10 +3437,11 @@ void sqlite3DefaultRowEst(Index *pIdx){
   int i;
 
   /* Set the first entry (number of rows in the index) to the estimated 
-  ** number of rows in the table. Or 10, if the estimated number of rows 
-  ** in the table is less than that.  */
+  ** number of rows in the table, or half the number of rows in the table
+  ** for a partial index.   But do not let the estimate drop below 10. */
   a[0] = pIdx->pTable->nRowLogEst;
-  if( a[0]<33 ) a[0] = 33;        assert( 33==sqlite3LogEst(10) );
+  if( pIdx->pPartIdxWhere!=0 ) a[0] -= 10;  assert( 10==sqlite3LogEst(2) );
+  if( a[0]<33 ) a[0] = 33;                  assert( 33==sqlite3LogEst(10) );
 
   /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is
   ** 6 and each subsequent value (if any) is 5.  */
@@ -4315,10 +4323,6 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){
 /*
 ** Return a KeyInfo structure that is appropriate for the given Index.
 **
-** The KeyInfo structure for an index is cached in the Index object.
-** So there might be multiple references to the returned pointer.  The
-** caller should not try to modify the KeyInfo object.
-**
 ** The caller should invoke sqlite3KeyInfoUnref() on the returned object
 ** when it has finished using it.
 */

src/ctime.c

@@ -71,7 +71,7 @@ static const char * const azCompileOpt[] = {
   "DISABLE_LFS",
 #endif
 #if SQLITE_ENABLE_8_3_NAMES
-  "ENABLE_8_3_NAMES",
+  "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES),
 #endif
 #if SQLITE_ENABLE_API_ARMOR
   "ENABLE_API_ARMOR",

src/dbstat.c

@@ -58,10 +58,10 @@
 */
 #define VTAB_SCHEMA                                                         \
   "CREATE TABLE xx( "                                                       \
-  "  name       STRING,           /* Name of table or index */"             \
-  "  path       INTEGER,          /* Path to page from root */"             \
+  "  name       TEXT,             /* Name of table or index */"             \
+  "  path       TEXT,             /* Path to page from root */"             \
   "  pageno     INTEGER,          /* Page number */"                        \
-  "  pagetype   STRING,           /* 'internal', 'leaf' or 'overflow' */"   \
+  "  pagetype   TEXT,             /* 'internal', 'leaf' or 'overflow' */"   \
   "  ncell      INTEGER,          /* Cells on page (0 for overflow) */"     \
   "  payload    INTEGER,          /* Bytes of payload on this page */"      \
   "  unused     INTEGER,          /* Bytes of unused space on this page */" \

src/expr.c

@@ -1818,6 +1818,11 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){
           eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0];
 
           if( prRhsHasNull && !pTab->aCol[iCol].notNull ){
+#ifdef SQLITE_ENABLE_COLUMN_USED_MASK
+            const i64 sOne = 1;
+            sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, 
+                iTab, 0, 0, (u8*)&sOne, P4_INT64);
+#endif
             *prRhsHasNull = ++pParse->nMem;
             sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull);
           }
@@ -2535,7 +2540,7 @@ void sqlite3ExprCodeGetColumnOfTable(
   }else{
     int op = IsVirtual(pTab) ? OP_VColumn : OP_Column;
     int x = iCol;
-    if( !HasRowid(pTab) ){
+    if( !HasRowid(pTab) && !IsVirtual(pTab) ){
       x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol);
     }
     sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut);

src/fkey.c

@@ -1372,7 +1372,8 @@ void sqlite3FkDelete(sqlite3 *db, Table *pTab){
   FKey *pFKey;                    /* Iterator variable */
   FKey *pNext;                    /* Copy of pFKey->pNextFrom */
 
-  assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) );
+  assert( db==0 || IsVirtual(pTab)
+         || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) );
   for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){
 
     /* Remove the FK from the fkeyHash hash table. */

src/func.c

@@ -740,7 +740,7 @@ static int patternCompare(
     }
     c2 = Utf8Read(zString);
     if( c==c2 ) continue;
-    if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){
+    if( noCase  && sqlite3Tolower(c)==sqlite3Tolower(c2) && c<0x80 && c2<0x80 ){
       continue;
     }
     if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue;

src/loadext.c

@@ -446,6 +446,7 @@ static int sqlite3LoadExtension(
   void **aHandle;
   u64 nMsg = 300 + sqlite3Strlen30(zFile);
   int ii;
+  int rc;
 
   /* Shared library endings to try if zFile cannot be loaded as written */
   static const char *azEndings[] = {
@@ -548,7 +549,9 @@ static int sqlite3LoadExtension(
     return SQLITE_ERROR;
   }
   sqlite3_free(zAltEntry);
-  if( xInit(db, &zErrmsg, &sqlite3Apis) ){
+  rc = xInit(db, &zErrmsg, &sqlite3Apis);
+  if( rc ){
+    if( rc==SQLITE_OK_LOAD_PERMANENTLY ) return SQLITE_OK;
     if( pzErrMsg ){
       *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg);
     }

src/os_win.c

@@ -5269,6 +5269,18 @@ static int winFullPathname(
   int nFull,                    /* Size of output buffer in bytes */
   char *zFull                   /* Output buffer */
 ){
+#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__)
+  DWORD nByte;
+  void *zConverted;
+  char *zOut;
+#endif
+
+  /* If this path name begins with "/X:", where "X" is any alphabetic
+  ** character, discard the initial "/" from the pathname.
+  */
+  if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){
+    zRelative++;
+  }
 
 #if defined(__CYGWIN__)
   SimulateIOError( return SQLITE_ERROR );
@@ -5347,17 +5359,6 @@ static int winFullPathname(
 #endif
 
 #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__)
-  DWORD nByte;
-  void *zConverted;
-  char *zOut;
-
-  /* If this path name begins with "/X:", where "X" is any alphabetic
-  ** character, discard the initial "/" from the pathname.
-  */
-  if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){
-    zRelative++;
-  }
-
   /* It's odd to simulate an io-error here, but really this is just
   ** using the io-error infrastructure to test that SQLite handles this
   ** function failing. This function could fail if, for example, the

src/parse.y

@@ -300,7 +300,8 @@ ccons ::= NULL onconf.
 ccons ::= NOT NULL onconf(R).    {sqlite3AddNotNull(pParse, R);}
 ccons ::= PRIMARY KEY sortorder(Z) onconf(R) autoinc(I).
                                  {sqlite3AddPrimaryKey(pParse,0,R,I,Z);}
-ccons ::= UNIQUE onconf(R).      {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0);}
+ccons ::= UNIQUE onconf(R).      {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0,
+                                   SQLITE_IDXTYPE_UNIQUE);}
 ccons ::= CHECK LP expr(X) RP.   {sqlite3AddCheckConstraint(pParse,X.pExpr);}
 ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R).
                                  {sqlite3CreateForeignKey(pParse,0,&T,TA,R);}
@@ -349,7 +350,8 @@ tcons ::= CONSTRAINT nm(X).      {pParse->constraintName = X;}
 tcons ::= PRIMARY KEY LP sortlist(X) autoinc(I) RP onconf(R).
                                  {sqlite3AddPrimaryKey(pParse,X,R,I,0);}
 tcons ::= UNIQUE LP sortlist(X) RP onconf(R).
-                                 {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0);}
+                                 {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0,
+                                       SQLITE_IDXTYPE_UNIQUE);}
 tcons ::= CHECK LP expr(E) RP onconf.
                                  {sqlite3AddCheckConstraint(pParse,E.pExpr);}
 tcons ::= FOREIGN KEY LP eidlist(FA) RP
@@ -1132,9 +1134,10 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
     exprNot(pParse, N, &A);
     A.zEnd = &E.z[E.n];
   }
-  expr(A) ::= expr(A) in_op(N) nm(Y) dbnm(Z). [IN] {
+  expr(A) ::= expr(A) in_op(N) nm(Y) dbnm(Z) paren_exprlist(E). [IN] {
     SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&Y,&Z);
     Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0);
+    if( E )  sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, E);
     A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0, 0);
     sqlite3PExprAddSelect(pParse, A.pExpr, pSelect);
     exprNot(pParse, N, &A);
@@ -1191,6 +1194,14 @@ nexprlist(A) ::= nexprlist(A) COMMA expr(Y).
 nexprlist(A) ::= expr(Y).
     {A = sqlite3ExprListAppend(pParse,0,Y.pExpr); /*A-overwrites-Y*/}
 
+/* A paren_exprlist is an optional expression list contained inside
+** of parenthesis */
+%type paren_exprlist {ExprList*}
+%destructor paren_exprlist {sqlite3ExprListDelete(pParse->db, $$);}
+paren_exprlist(A) ::= .   {A = 0;}
+paren_exprlist(A) ::= LP exprlist(X) RP.  {A = X;}
+
+
 
 ///////////////////////////// The CREATE INDEX command ///////////////////////
 //
@@ -1198,7 +1209,7 @@ cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D)
         ON nm(Y) LP sortlist(Z) RP where_opt(W). {
   sqlite3CreateIndex(pParse, &X, &D, 
                      sqlite3SrcListAppend(pParse->db,0,&Y,0), Z, U,
-                      &S, W, SQLITE_SO_ASC, NE);
+                      &S, W, SQLITE_SO_ASC, NE, SQLITE_IDXTYPE_APPDEF);
 }
 
 %type uniqueflag {int}

src/pragma.c

@@ -1052,7 +1052,7 @@ void sqlite3Pragma(
   */
   case PragTyp_TABLE_INFO: if( zRight ){
     Table *pTab;
-    pTab = sqlite3FindTable(db, zRight, zDb);
+    pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb);
     if( pTab ){
       static const char *azCol[] = {
          "cid", "name", "type", "notnull", "dflt_value", "pk"

src/select.c

@@ -1028,7 +1028,7 @@ static void selectInnerLoop(
 */
 KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){
   int nExtra = (N+X)*(sizeof(CollSeq*)+1);
-  KeyInfo *p = sqlite3Malloc(sizeof(KeyInfo) + nExtra);
+  KeyInfo *p = sqlite3DbMallocRaw(db, sizeof(KeyInfo) + nExtra);
   if( p ){
     p->aSortOrder = (u8*)&p->aColl[N+X];
     p->nField = (u16)N;
@@ -1050,7 +1050,7 @@ void sqlite3KeyInfoUnref(KeyInfo *p){
   if( p ){
     assert( p->nRef>0 );
     p->nRef--;
-    if( p->nRef==0 ) sqlite3DbFree(0, p);
+    if( p->nRef==0 ) sqlite3DbFree(p->db, p);
   }
 }
 

src/sqlite.h.in

@@ -506,6 +506,7 @@ int sqlite3_exec(
 #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8))
 #define SQLITE_WARNING_AUTOINDEX       (SQLITE_WARNING | (1<<8))
 #define SQLITE_AUTH_USER               (SQLITE_AUTH | (1<<8))
+#define SQLITE_OK_LOAD_PERMANENTLY     (SQLITE_OK | (1<<8))
 
 /*
 ** CAPI3REF: Flags For File Open Operations

src/sqliteInt.h

@@ -2485,7 +2485,7 @@ struct SrcList {
     int regReturn;    /* Register holding return address of addrFillSub */
     int regResult;    /* Registers holding results of a co-routine */
     struct {
-      u8 jointype;      /* Type of join between this able and the previous */
+      u8 jointype;      /* Type of join between this table and the previous */
       unsigned notIndexed :1;    /* True if there is a NOT INDEXED clause */
       unsigned isIndexedBy :1;   /* True if there is an INDEXED BY clause */
       unsigned isTabFunc :1;     /* True if table-valued-function syntax */
@@ -3617,8 +3617,8 @@ void sqlite3SrcListAssignCursors(Parse*, SrcList*);
 void sqlite3IdListDelete(sqlite3*, IdList*);
 void sqlite3SrcListDelete(sqlite3*, SrcList*);
 Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**);
-Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
-                          Expr*, int, int);
+void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
+                          Expr*, int, int, u8);
 void sqlite3DropIndex(Parse*, SrcList*, int);
 int sqlite3Select(Parse*, Select*, SelectDest*);
 Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*,
@@ -3671,8 +3671,10 @@ void sqlite3ExprIfTrue(Parse*, Expr*, int, int);
 void sqlite3ExprIfFalse(Parse*, Expr*, int, int);
 void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int);
 Table *sqlite3FindTable(sqlite3*,const char*, const char*);
-Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*);
-Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *);
+#define LOCATE_VIEW    0x01
+#define LOCATE_NOERR   0x02
+Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
+Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
 Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
 void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
 void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);

src/tclsqlite.c

@@ -2325,6 +2325,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   */
   case DB_EXISTS: 
   case DB_ONECOLUMN: {
+    Tcl_Obj *pResult = 0;
     DbEvalContext sEval;
     if( objc!=3 ){
       Tcl_WrongNumArgs(interp, 2, objv, "SQL");
@@ -2335,14 +2336,15 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     rc = dbEvalStep(&sEval);
     if( choice==DB_ONECOLUMN ){
       if( rc==TCL_OK ){
-        Tcl_SetObjResult(interp, dbEvalColumnValue(&sEval, 0));
+        pResult = dbEvalColumnValue(&sEval, 0);
       }else if( rc==TCL_BREAK ){
         Tcl_ResetResult(interp);
       }
     }else if( rc==TCL_BREAK || rc==TCL_OK ){
-      Tcl_SetObjResult(interp, Tcl_NewBooleanObj(rc==TCL_OK));
+      pResult = Tcl_NewBooleanObj(rc==TCL_OK);
     }
     dbEvalFinalize(&sEval);
+    if( pResult ) Tcl_SetObjResult(interp, pResult);
 
     if( rc==TCL_BREAK ){
       rc = TCL_OK;

src/test1.c

@@ -3241,6 +3241,142 @@ static int test_bind_int(
 }
 
 
+/*
+** Usage:   intarray_addr  INT  ...
+**
+** Return the address of a C-language array of 32-bit integers.
+**
+** Space to hold the array is obtained from malloc().  Call this procedure once
+** with no arguments in order to release memory.  Each call to this procedure
+** overwrites the previous array.
+*/
+static int test_intarray_addr(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int i;
+  static int *p = 0;
+
+  sqlite3_free(p);
+  p = 0;
+  if( objc>1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; i<objc-1; i++){
+      if( Tcl_GetIntFromObj(interp, objv[1+i], &p[i]) ){
+        sqlite3_free(p);
+        p = 0;
+        return TCL_ERROR;
+      }
+    }
+  }  
+  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((sqlite3_int64)p));
+  return TCL_OK;
+}
+/*
+** Usage:   intarray_addr  INT  ...
+**
+** Return the address of a C-language array of 32-bit integers.
+**
+** Space to hold the array is obtained from malloc().  Call this procedure once
+** with no arguments in order to release memory.  Each call to this procedure
+** overwrites the previous array.
+*/
+static int test_int64array_addr(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int i;
+  static sqlite3_int64 *p = 0;
+
+  sqlite3_free(p);
+  p = 0;
+  if( objc>1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; i<objc-1; i++){
+      if( Tcl_GetWideIntFromObj(interp, objv[1+i], &p[i]) ){
+        sqlite3_free(p);
+        p = 0;
+        return TCL_ERROR;
+      }
+    }
+  }  
+  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((sqlite3_int64)p));
+  return TCL_OK;
+}
+/*
+** Usage:   doublearray_addr  INT  ...
+**
+** Return the address of a C-language array of doubles.
+**
+** Space to hold the array is obtained from malloc().  Call this procedure once
+** with no arguments in order to release memory.  Each call to this procedure
+** overwrites the previous array.
+*/
+static int test_doublearray_addr(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int i;
+  static double *p = 0;
+
+  sqlite3_free(p);
+  p = 0;
+  if( objc>1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; i<objc-1; i++){
+      if( Tcl_GetDoubleFromObj(interp, objv[1+i], &p[i]) ){
+        sqlite3_free(p);
+        p = 0;
+        return TCL_ERROR;
+      }
+    }
+  }  
+  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((sqlite3_int64)p));
+  return TCL_OK;
+}
+/*
+** Usage:   textarray_addr  TEXT ...
+**
+** Return the address of a C-language array of strings.
+**
+** Space to hold the array is obtained from malloc().  Call this procedure once
+** with no arguments in order to release memory.  Each call to this procedure
+** overwrites the previous array.
+*/
+static int test_textarray_addr(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int i;
+  static int n = 0;
+  static char **p = 0;
+
+  for(i=0; i<n; i++) sqlite3_free(p[i]);
+  sqlite3_free(p);
+  p = 0;
+  if( objc>1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; i<objc-1; i++){
+      p[i] = sqlite3_mprintf("%s", Tcl_GetString(objv[1+i]));
+    }
+  }  
+  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((sqlite3_int64)p));
+  return TCL_OK;
+}
+
+
 /*
 ** Usage:   sqlite3_bind_int64  STMT N VALUE
 **
@@ -6583,7 +6719,9 @@ static int tclLoadStaticExtensionCmd(
   Tcl_Obj *CONST objv[]
 ){
   extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_carray_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_csv_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_eval_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_fileio_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*);
@@ -6600,7 +6738,9 @@ static int tclLoadStaticExtensionCmd(
     int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*);
   } aExtension[] = {
     { "amatch",                sqlite3_amatch_init               },
+    { "carray",                sqlite3_carray_init               },
     { "closure",               sqlite3_closure_init              },
+    { "csv",                   sqlite3_csv_init                  },
     { "eval",                  sqlite3_eval_init                 },
     { "fileio",                sqlite3_fileio_init               },
     { "fuzzer",                sqlite3_fuzzer_init               },
@@ -7093,6 +7233,10 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "bad_behavior",                  test_bad_behavior,  (void*)&iZero },
      { "register_dbstat_vtab",          test_register_dbstat_vtab  },
      { "sqlite3_connection_pointer",    get_sqlite_pointer, 0 },
+     { "intarray_addr",                 test_intarray_addr, 0 },
+     { "int64array_addr",               test_int64array_addr, 0 },
+     { "doublearray_addr",              test_doublearray_addr, 0 },
+     { "textarray_addr",                test_textarray_addr, 0 },
      { "sqlite3_bind_int",              test_bind_int,      0 },
      { "sqlite3_bind_zeroblob",         test_bind_zeroblob, 0 },
      { "sqlite3_bind_zeroblob64",       test_bind_zeroblob64, 0 },

src/test3.c

@@ -385,8 +385,7 @@ static int btree_payload_size(
   const char **argv      /* Text of each argument */
 ){
   BtCursor *pCur;
-  int n2;
-  u64 n1;
+  u32 n;
   char zBuf[50];
 
   if( argc!=2 ){
@@ -396,17 +395,9 @@ static int btree_payload_size(
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
   sqlite3BtreeEnter(pCur->pBtree);
-
-  /* The cursor may be in "require-seek" state. If this is the case, the
-  ** call to BtreeDataSize() will fix it. */
-  sqlite3BtreeDataSize(pCur, (u32*)&n2);
-  if( pCur->apPage[pCur->iPage]->intKey ){
-    n1 = 0;
-  }else{
-    sqlite3BtreeKeySize(pCur, (i64*)&n1);
-  }
+  n = sqlite3BtreePayloadSize(pCur);
   sqlite3BtreeLeave(pCur->pBtree);
-  sqlite3_snprintf(sizeof(zBuf),zBuf, "%d", (int)(n1+n2));
+  sqlite3_snprintf(sizeof(zBuf),zBuf, "%u", n);
   Tcl_AppendResult(interp, zBuf, 0);
   return SQLITE_OK;
 }

src/vdbe.c

@@ -2016,6 +2016,7 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
       if( (flags1 | flags3)&MEM_Str ){
         if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
           applyNumericAffinity(pIn1,0);
+          flags3 = pIn3->flags;
         }
         if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
           applyNumericAffinity(pIn3,0);
@@ -2028,6 +2029,7 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
         sqlite3VdbeMemStringify(pIn1, encoding, 1);
         testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) );
         flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask);
+        flags3 = pIn3->flags;
       }
       if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){
         testcase( pIn3->flags & MEM_Int );
@@ -2380,7 +2382,6 @@ case OP_NotNull: {            /* same as TK_NOTNULL, jump, in1 */
 ** skipped for length() and all content loading can be skipped for typeof().
 */
 case OP_Column: {
-  i64 payloadSize64; /* Number of bytes in the record */
   int p2;            /* column number to retrieve */
   VdbeCursor *pC;    /* The VDBE cursor */
   BtCursor *pCrsr;   /* The BTree cursor */
@@ -2433,22 +2434,9 @@ case OP_Column: {
     }else{
       assert( pC->eCurType==CURTYPE_BTREE );
       assert( pCrsr );
-      if( pC->isTable==0 ){
-        assert( sqlite3BtreeCursorIsValid(pCrsr) );
-        VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64);
-        assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */
-        /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the
-        ** payload size, so it is impossible for payloadSize64 to be
-        ** larger than 32 bits. */
-        assert( (payloadSize64 & SQLITE_MAX_U32)==(u64)payloadSize64 );
-        pC->aRow = sqlite3BtreeKeyFetch(pCrsr, &avail);
-        pC->payloadSize = (u32)payloadSize64;
-      }else{
-        assert( sqlite3BtreeCursorIsValid(pCrsr) );
-        VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &pC->payloadSize);
-        assert( rc==SQLITE_OK );   /* DataSize() cannot fail */
-        pC->aRow = sqlite3BtreeDataFetch(pCrsr, &avail);
-      }
+      assert( sqlite3BtreeCursorIsValid(pCrsr) );
+      pC->payloadSize = sqlite3BtreePayloadSize(pCrsr);
+      pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &avail);
       assert( avail<=65536 );  /* Maximum page size is 64KiB */
       if( pC->payloadSize <= (u32)avail ){
         pC->szRow = pC->payloadSize;
@@ -4201,8 +4189,7 @@ case OP_NewRowid: {           /* out2 */
         v = 1;   /* IMP: R-61914-48074 */
       }else{
         assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) );
-        rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
-        assert( rc==SQLITE_OK );   /* Cannot fail following BtreeLast() */
+        v = sqlite3BtreeIntegerKey(pC->uc.pCursor);
         if( v>=MAX_ROWID ){
           pC->useRandomRowid = 1;
         }else{
@@ -4459,8 +4446,7 @@ case OP_Delete: {
     /* If p5 is zero, the seek operation that positioned the cursor prior to
     ** OP_Delete will have also set the pC->movetoTarget field to the rowid of
     ** the row that is being deleted */
-    i64 iKey = 0;
-    sqlite3BtreeKeySize(pC->uc.pCursor, &iKey);
+    i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor);
     assert( pC->movetoTarget==iKey );
   }
 #endif
@@ -4476,7 +4462,7 @@ case OP_Delete: {
     zDb = db->aDb[pC->iDb].zName;
     pTab = pOp->p4.pTab;
     if( (pOp->p5 & OPFLAG_SAVEPOSITION)!=0 && pC->isTable ){
-      sqlite3BtreeKeySize(pC->uc.pCursor, &pC->movetoTarget);
+      pC->movetoTarget = sqlite3BtreeIntegerKey(pC->uc.pCursor);
     }
   }else{
     zDb = 0;   /* Not needed.  Silence a compiler warning. */
@@ -4630,7 +4616,6 @@ case OP_RowData: {
   VdbeCursor *pC;
   BtCursor *pCrsr;
   u32 n;
-  i64 n64;
 
   pOut = &aMem[pOp->p2];
   memAboutToChange(p, pOut);
@@ -4662,20 +4647,9 @@ case OP_RowData: {
   if( rc!=SQLITE_OK ) goto abort_due_to_error;
 #endif
 
-  if( pC->isTable==0 ){
-    assert( !pC->isTable );
-    VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &n64);
-    assert( rc==SQLITE_OK );    /* True because of CursorMoveto() call above */
-    if( n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){
-      goto too_big;
-    }
-    n = (u32)n64;
-  }else{
-    VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &n);
-    assert( rc==SQLITE_OK );    /* DataSize() cannot fail */
-    if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
-      goto too_big;
-    }
+  n = sqlite3BtreePayloadSize(pCrsr);
+  if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
+    goto too_big;
   }
   testcase( n==0 );
   if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){
@@ -4740,8 +4714,7 @@ case OP_Rowid: {                 /* out2 */
       pOut->flags = MEM_Null;
       break;
     }
-    rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
-    assert( rc==SQLITE_OK );  /* Always so because of CursorRestore() above */
+    v = sqlite3BtreeIntegerKey(pC->uc.pCursor);
   }
   pOut->u.i = v;
   break;

src/vdbeapi.c

@@ -1650,8 +1650,7 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
     u32 nRec;
     u8 *aRec;
 
-    rc = sqlite3BtreeDataSize(p->pCsr->uc.pCursor, &nRec);
-    if( rc!=SQLITE_OK ) goto preupdate_old_out;
+    nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor);
     aRec = sqlite3DbMallocRaw(db, nRec);
     if( !aRec ) goto preupdate_old_out;
     rc = sqlite3BtreeData(p->pCsr->uc.pCursor, 0, nRec, aRec);

src/vdbeaux.c

@@ -4315,8 +4315,7 @@ int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){
   ** this code can safely assume that nCellKey is 32-bits  
   */
   assert( sqlite3BtreeCursorIsValid(pCur) );
-  VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey);
-  assert( rc==SQLITE_OK );     /* pCur is always valid so KeySize cannot fail */
+  nCellKey = sqlite3BtreePayloadSize(pCur);
   assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey );
 
   /* Read in the complete content of the index entry */
@@ -4393,8 +4392,7 @@ int sqlite3VdbeIdxKeyCompare(
   assert( pC->eCurType==CURTYPE_BTREE );
   pCur = pC->uc.pCursor;
   assert( sqlite3BtreeCursorIsValid(pCur) );
-  VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey);
-  assert( rc==SQLITE_OK );    /* pCur is always valid so KeySize cannot fail */
+  nCellKey = sqlite3BtreePayloadSize(pCur);
   /* nCellKey will always be between 0 and 0xffffffff because of the way
   ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */
   if( nCellKey<=0 || nCellKey>0x7fffffff ){

src/vdbeblob.c

@@ -415,7 +415,7 @@ static int blobReadWrite(
       ** anyhow.
       */
       sqlite3_int64 iKey;
-      sqlite3BtreeKeySize(p->pCsr, &iKey);
+      iKey = sqlite3BtreeIntegerKey(p->pCsr);
       sqlite3VdbePreUpdateHook(
           v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1
       );

src/vdbemem.c

@@ -991,11 +991,7 @@ int sqlite3VdbeMemFromBtree(
   /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() 
   ** that both the BtShared and database handle mutexes are held. */
   assert( (pMem->flags & MEM_RowSet)==0 );
-  if( key ){
-    zData = (char *)sqlite3BtreeKeyFetch(pCur, &available);
-  }else{
-    zData = (char *)sqlite3BtreeDataFetch(pCur, &available);
-  }
+  zData = (char *)sqlite3BtreePayloadFetch(pCur, &available);
   assert( zData!=0 );
 
   if( offset+amt<=available ){

src/vtab.c

@@ -754,10 +754,24 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
      && (pParse->pNewTable->tabFlags & TF_Virtual)==0
     ){
       if( !pTab->aCol ){
-        pTab->aCol = pParse->pNewTable->aCol;
-        pTab->nCol = pParse->pNewTable->nCol;
-        pParse->pNewTable->nCol = 0;
-        pParse->pNewTable->aCol = 0;
+        Table *pNew = pParse->pNewTable;
+        Index *pIdx;
+        pTab->aCol = pNew->aCol;
+        pTab->nCol = pNew->nCol;
+        pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid);
+        pNew->nCol = 0;
+        pNew->aCol = 0;
+        assert( pTab->pIndex==0 );
+        if( !HasRowid(pNew) && pCtx->pVTable->pMod->pModule->xUpdate!=0 ){
+          rc = SQLITE_ERROR;
+        }
+        pIdx = pNew->pIndex;
+        if( pIdx ){
+          assert( pIdx->pNext==0 );
+          pTab->pIndex = pIdx;
+          pNew->pIndex = 0;
+          pIdx->pTable = pTab;
+        }
       }
       pCtx->bDeclared = 1;
     }else{
@@ -1087,7 +1101,7 @@ void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){
 }
 
 /*
-** Check to see if virtual tale module pMod can be have an eponymous
+** Check to see if virtual table module pMod can be have an eponymous
 ** virtual table instance.  If it can, create one if one does not already
 ** exist. Return non-zero if the eponymous virtual table instance exists
 ** when this routine returns, and return zero if it does not exist.
@@ -1104,17 +1118,18 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
   const sqlite3_module *pModule = pMod->pModule;
   Table *pTab;
   char *zErr = 0;
-  int nName;
   int rc;
   sqlite3 *db = pParse->db;
   if( pMod->pEpoTab ) return 1;
   if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0;
-  nName = sqlite3Strlen30(pMod->zName) + 1;
-  pTab = sqlite3DbMallocZero(db, sizeof(Table) + nName);
+  pTab = sqlite3DbMallocZero(db, sizeof(Table));
   if( pTab==0 ) return 0;
+  pTab->zName = sqlite3DbStrDup(db, pMod->zName);
+  if( pTab->zName==0 ){
+    sqlite3DbFree(db, pTab);
+    return 0;
+  }
   pMod->pEpoTab = pTab;
-  pTab->zName = (char*)&pTab[1];
-  memcpy(pTab->zName, pMod->zName, nName);
   pTab->nRef = 1;
   pTab->pSchema = db->aDb[0].pSchema;
   pTab->tabFlags |= TF_Virtual;
@@ -1140,9 +1155,11 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
 void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){
   Table *pTab = pMod->pEpoTab;
   if( pTab!=0 ){
-    sqlite3DeleteColumnNames(db, pTab);
-    sqlite3VtabClear(db, pTab);
-    sqlite3DbFree(db, pTab);
+    /* Mark the table as Ephemeral prior to deleting it, so that the
+    ** sqlite3DeleteTable() routine will know that it is not stored in 
+    ** the schema. */
+    pTab->tabFlags |= TF_Ephemeral;
+    sqlite3DeleteTable(db, pTab);
     pMod->pEpoTab = 0;
   }
 }

src/where.c

@@ -2701,6 +2701,7 @@ static int whereLoopAddBtree(
           pNew->rSetup += 24;
         }
         ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
+        if( pNew->rSetup<0 ) pNew->rSetup = 0;
         /* TUNING: Each index lookup yields 20 rows in the table.  This
         ** is more than the usual guess of 10 rows, since we have no way
         ** of knowing how selective the index will ultimately be.  It would
@@ -2761,6 +2762,7 @@ static int whereLoopAddBtree(
       /* Full scan via index */
       if( b
        || !HasRowid(pTab)
+       || pProbe->pPartIdxWhere!=0
        || ( m==0
          && pProbe->bUnordered==0
          && (pProbe->szIdxRow<pTab->szTabRow)
@@ -3239,6 +3241,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
       mPrereq = mPrior;
     }
     priorJointype = pItem->fg.jointype;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
     if( IsVirtual(pItem->pTab) ){
       struct SrcList_item *p;
       for(p=&pItem[1]; p<pEnd; p++){
@@ -3247,7 +3250,9 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
         }
       }
       rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
-    }else{
+    }else
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+    {
       rc = whereLoopAddBtree(pBuilder, mPrereq);
     }
     if( rc==SQLITE_OK ){

src/wherecode.c

@@ -626,6 +626,38 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){
   return WRC_Continue;
 }
 
+/*
+** Test whether or not expression pExpr, which was part of a WHERE clause,
+** should be included in the cursor-hint for a table that is on the rhs
+** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the 
+** expression is not suitable.
+**
+** An expression is unsuitable if it might evaluate to non NULL even if
+** a TK_COLUMN node that does affect the value of the expression is set
+** to NULL. For example:
+**
+**   col IS NULL
+**   col IS NOT NULL
+**   coalesce(col, 1)
+**   CASE WHEN col THEN 0 ELSE 1 END
+*/
+static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){
+  if( pExpr->op==TK_IS 
+   || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT 
+   || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE 
+  ){
+    pWalker->eCode = 1;
+  }else if( pExpr->op==TK_FUNCTION ){
+    int d1;
+    char d2[3];
+    if( 0==sqlite3IsLikeFunction(pWalker->pParse->db, pExpr, &d1, d2) ){
+      pWalker->eCode = 1;
+    }
+  }
+
+  return WRC_Continue;
+}
+
 
 /*
 ** This function is called on every node of an expression tree used as an
@@ -678,6 +710,7 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
 ** Insert an OP_CursorHint instruction if it is appropriate to do so.
 */
 static void codeCursorHint(
+  struct SrcList_item *pTabItem,  /* FROM clause item */
   WhereInfo *pWInfo,    /* The where clause */
   WhereLevel *pLevel,   /* Which loop to provide hints for */
   WhereTerm *pEndRange  /* Hint this end-of-scan boundary term if not NULL */
@@ -708,7 +741,42 @@ static void codeCursorHint(
     pTerm = &pWC->a[i];
     if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
     if( pTerm->prereqAll & pLevel->notReady ) continue;
-    if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
+
+    /* Any terms specified as part of the ON(...) clause for any LEFT 
+    ** JOIN for which the current table is not the rhs are omitted
+    ** from the cursor-hint. 
+    **
+    ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms 
+    ** that were specified as part of the WHERE clause must be excluded.
+    ** This is to address the following:
+    **
+    **   SELECT ... t1 LEFT JOIN t2 ON (t1.a=t2.b) WHERE t2.c IS NULL;
+    **
+    ** Say there is a single row in t2 that matches (t1.a=t2.b), but its
+    ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is 
+    ** pushed down to the cursor, this row is filtered out, causing
+    ** SQLite to synthesize a row of NULL values. Which does match the
+    ** WHERE clause, and so the query returns a row. Which is incorrect.
+    **
+    ** For the same reason, WHERE terms such as:
+    **
+    **   WHERE 1 = (t2.c IS NULL)
+    **
+    ** are also excluded. See codeCursorHintIsOrFunction() for details.
+    */
+    if( pTabItem->fg.jointype & JT_LEFT ){
+      Expr *pExpr = pTerm->pExpr;
+      if( !ExprHasProperty(pExpr, EP_FromJoin) 
+       || pExpr->iRightJoinTable!=pTabItem->iCursor
+      ){
+        sWalker.eCode = 0;
+        sWalker.xExprCallback = codeCursorHintIsOrFunction;
+        sqlite3WalkExpr(&sWalker, pTerm->pExpr);
+        if( sWalker.eCode ) continue;
+      }
+    }else{
+      if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
+    }
 
     /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize
     ** the cursor.  These terms are not needed as hints for a pure range
@@ -742,7 +810,7 @@ static void codeCursorHint(
   }
 }
 #else
-# define codeCursorHint(A,B,C)  /* No-op */
+# define codeCursorHint(A,B,C,D)  /* No-op */
 #endif /* SQLITE_ENABLE_CURSOR_HINTS */
 
 /*
@@ -998,7 +1066,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       pStart = pEnd;
       pEnd = pTerm;
     }
-    codeCursorHint(pWInfo, pLevel, pEnd);
+    codeCursorHint(pTabItem, pWInfo, pLevel, pEnd);
     if( pStart ){
       Expr *pX;             /* The expression that defines the start bound */
       int r1, rTemp;        /* Registers for holding the start boundary */
@@ -1212,7 +1280,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     ** and store the values of those terms in an array of registers
     ** starting at regBase.
     */
-    codeCursorHint(pWInfo, pLevel, pRangeEnd);
+    codeCursorHint(pTabItem, pWInfo, pLevel, pRangeEnd);
     regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff);
     assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq );
     if( zStartAff ) cEndAff = zStartAff[nEq];
@@ -1660,7 +1728,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       ** a pseudo-cursor.  No need to Rewind or Next such cursors. */
       pLevel->op = OP_Noop;
     }else{
-      codeCursorHint(pWInfo, pLevel, 0);
+      codeCursorHint(pTabItem, pWInfo, pLevel, 0);
       pLevel->op = aStep[bRev];
       pLevel->p1 = iCur;
       pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);

src/whereexpr.c

@@ -568,7 +568,9 @@ static void exprAnalyzeOrTerm(
         if( !db->mallocFailed ){
           for(j=0, pAndTerm=pAndWC->a; j<pAndWC->nTerm; j++, pAndTerm++){
             assert( pAndTerm->pExpr );
-            if( allowedOp(pAndTerm->pExpr->op) ){
+            if( allowedOp(pAndTerm->pExpr->op) 
+             || pAndTerm->eOperator==WO_MATCH 
+            ){
               b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor);
             }
           }
@@ -783,12 +785,10 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr){
   pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight);
   if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1;
   pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft);
-  /* Since pLeft and pRight are both a column references, their collating
-  ** sequence should always be defined. */
-  zColl1 = ALWAYS(pColl) ? pColl->zName : 0;
+  zColl1 = pColl ? pColl->zName : 0;
   pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight);
-  zColl2 = ALWAYS(pColl) ? pColl->zName : 0;
-  return sqlite3StrICmp(zColl1, zColl2)==0;
+  zColl2 = pColl ? pColl->zName : 0;
+  return sqlite3_stricmp(zColl1, zColl2)==0;
 }
 
 /*
@@ -1122,7 +1122,7 @@ static void exprAnalyze(
   ** virtual tables.  The native query optimizer does not attempt
   ** to do anything with MATCH functions.
   */
-  if( isMatchOfColumn(pExpr, &eOp2) ){
+  if( pWC->op==TK_AND && isMatchOfColumn(pExpr, &eOp2) ){
     int idxNew;
     Expr *pRight, *pLeft;
     WhereTerm *pNewTerm;

test/bestindex3.test

@@ -0,0 +1,170 @@
+# 2016 May 29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindex3
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Virtual table callback for a virtual table named $tbl.
+#
+# The table created is:
+#
+#      "CREATE TABLE t1 (a, b, c)"
+#
+# This virtual table supports both LIKE and = operators on all columns.
+#  
+proc vtab_cmd {bOmit method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c)"
+    }
+
+    xBestIndex {
+      foreach {clist orderby mask} $args {}
+
+      set ret [list]
+      set use use
+      if {$bOmit} {set use omit}
+
+      for {set i 0} {$i < [llength $clist]} {incr i} {
+        array unset C
+        array set C [lindex $clist $i]
+        if {$C(usable) && ($C(op)=="like" || $C(op)=="eq")} {
+          lappend ret $use $i
+          lappend ret idxstr 
+          lappend ret "[lindex {a b c} $C(column)] [string toupper $C(op)] ?"
+          break
+        }
+      }
+
+      if {$ret==""} {
+        lappend ret cost 1000000 rows 1000000
+      } else {
+        lappend ret cost 100 rows 10
+      }
+      return $ret
+    }
+
+    xFilter {
+      foreach {idxnum idxstr param} $args {}
+      set where ""
+      if {$bOmit && $idxstr != ""} {
+        set where " WHERE [string map [list ? '$param' EQ =] $idxstr]"
+      }
+      return [list sql "SELECT rowid, * FROM ttt$where"]
+    }
+  }
+  return ""
+}
+
+register_tcl_module db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING tcl("vtab_cmd 0");
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM t1 WHERE a LIKE 'abc';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
+}
+
+do_eqp_test 1.2 {
+  SELECT * FROM t1 WHERE a = 'abc';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
+}
+
+do_eqp_test 1.3 {
+  SELECT * FROM t1 WHERE a = 'abc' OR b = 'def';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
+}
+
+do_eqp_test 1.4 {
+  SELECT * FROM t1 WHERE a LIKE 'abc%' OR b = 'def';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
+}
+
+do_execsql_test 1.5 {
+  CREATE TABLE ttt(a, b, c);
+
+  INSERT INTO ttt VALUES(1, 'two',   'three');
+  INSERT INTO ttt VALUES(2, 'one',   'two');
+  INSERT INTO ttt VALUES(3, 'three', 'one');
+  INSERT INTO ttt VALUES(4, 'y',     'one');
+  INSERT INTO ttt VALUES(5, 'x',     'two');
+  INSERT INTO ttt VALUES(6, 'y',     'three');
+}
+
+foreach omit {0 1} {
+  do_execsql_test 1.6.$omit.0 "
+    DROP TABLE t1;
+    CREATE VIRTUAL TABLE t1 USING tcl('vtab_cmd $omit');
+  "
+  do_execsql_test 1.6.$omit.1 { 
+    SELECT rowid FROM t1 WHERE c LIKE 'o%'
+  } {3 4}
+
+  do_execsql_test 1.6.$omit.2 { 
+    SELECT rowid FROM t1 WHERE c LIKE 'o%' OR b='y'
+  } {3 4 6}
+
+  do_execsql_test 1.6.$omit.3 { 
+    SELECT rowid FROM t1 WHERE c = 'three' OR c LIKE 'o%'
+  } {1 6 3 4}
+}
+
+#-------------------------------------------------------------------------
+# Test the same pattern works with ordinary tables.
+#
+do_execsql_test 2.1 {
+  CREATE TABLE t2(x TEXT COLLATE nocase, y TEXT);
+  CREATE INDEX t2x ON t2(x COLLATE nocase);
+  CREATE INDEX t2y ON t2(y);
+}
+
+do_eqp_test 2.2 {
+  SELECT * FROM t2 WHERE x LIKE 'abc%' OR y = 'def'
+} {
+  0 0 0 {SEARCH TABLE t2 USING INDEX t2x (x>? AND x<?)}
+  0 0 0 {SEARCH TABLE t2 USING INDEX t2y (y=?)}
+}
+
+#-------------------------------------------------------------------------
+# Test that any PRIMARY KEY within a sqlite3_decl_vtab() CREATE TABLE 
+# statement is currently ignored.
+#
+proc vvv_command {method args} {
+  switch -- $method {
+    xConnect { return "CREATE TABLE t1(a PRIMARY KEY, b, c)" }
+  }
+}
+proc yyy_command {method args} {
+  switch -- $method {
+    xConnect { return "CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b))" }
+  }
+}
+
+do_execsql_test 3.1 { CREATE VIRTUAL TABLE t3 USING tcl('vvv_command') }
+do_execsql_test 3.2 { CREATE VIRTUAL TABLE t4 USING tcl('yyy_command') }
+
+finish_test
+

test/collateB.test

@@ -0,0 +1,63 @@
+# 2016-07-01
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Test cases for a crash bug.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test collateB-1.1 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY);
+  CREATE TABLE t2(b INTEGER PRIMARY KEY, x1 INT COLLATE NOCASE);
+  CREATE TABLE t3(x2 INT);
+  SELECT * FROM t3, t2, t1 WHERE x2=b AND x1=a AND a=1;
+} {}
+do_execsql_test collateB-1.2 {
+  INSERT INTO t1(a) VALUES(1),(2),(3);
+  INSERT INTO t2(b,x1) VALUES(11,1),(22,2),(33,3);
+  INSERT INTO t3(x2) VALUES(11),(22),(33);
+  SELECT *,'|' FROM t3, t2, t1 WHERE x2=b AND x1=a AND a=1;
+} {11 11 1 1 |}
+do_execsql_test collateB-1.3 {
+  SELECT *,'|' FROM t3, t1, t2 WHERE x2=b AND x1=a AND a=1;
+} {11 1 11 1 |}
+do_execsql_test collateB-1.4 {
+  SELECT *,'|' FROM t2, t3, t1 WHERE x2=b AND x1=a AND a=1;
+} {11 1 11 1 |}
+do_execsql_test collateB-1.5 {
+  SELECT *,'|' FROM t2, t1, t3 WHERE x2=b AND x1=a AND a=1;
+} {11 1 1 11 |}
+do_execsql_test collateB-1.6 {
+  SELECT *,'|' FROM t1, t2, t3 WHERE x2=b AND x1=a AND a=1;
+} {1 11 1 11 |}
+do_execsql_test collateB-1.7 {
+  SELECT *,'|' FROM t1, t2, t3 WHERE x2=b AND x1=a AND a=1;
+} {1 11 1 11 |}
+do_execsql_test collateB-1.12 {
+  SELECT *,'|' FROM t3, t2, t1 WHERE b=x2 AND a=x1 AND 1=a;
+} {11 11 1 1 |}
+do_execsql_test collateB-1.13 {
+  SELECT *,'|' FROM t3, t1, t2 WHERE b=x2 AND a=x1 AND 1=a;
+} {11 1 11 1 |}
+do_execsql_test collateB-1.14 {
+  SELECT *,'|' FROM t2, t3, t1 WHERE b=x2 AND a=x1 AND 1=a;
+} {11 1 11 1 |}
+do_execsql_test collateB-1.15 {
+  SELECT *,'|' FROM t2, t1, t3 WHERE b=x2 AND a=x1 AND 1=a;
+} {11 1 1 11 |}
+do_execsql_test collateB-1.16 {
+  SELECT *,'|' FROM t1, t2, t3 WHERE b=x2 AND a=x1 AND 1=a;
+} {1 11 1 11 |}
+do_execsql_test collateB-1.17 {
+  SELECT *,'|' FROM t1, t2, t3 WHERE b=x2 AND a=x1 AND 1=a;
+} {1 11 1 11 |}
+
+finish_test

test/csv01.test

@@ -0,0 +1,112 @@
+# 2016-06-02
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# Test cases for CSV virtual table.
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix closure01
+
+ifcapable !vtab||!cte { finish_test ; return }
+
+load_static_extension db csv
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE temp.t1 USING csv(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4
+  );
+  SELECT * FROM t1 WHERE c1=10;
+} {9 10 11 12}
+do_execsql_test 1.1 {
+  SELECT * FROM t1 WHERE c1='10';
+} {9 10 11 12}
+do_execsql_test 1.2 {
+  SELECT rowid FROM t1;
+} {1 2 3 4}
+
+do_execsql_test 2.0 {
+  DROP TABLE t1;
+  CREATE VIRTUAL TABLE temp.t2 USING csv(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4,
+    schema='CREATE TABLE t2(a INT, b TEXT, c REAL, d BLOB)'
+  );
+  SELECT * FROM t2 WHERE a=9;
+} {9 10 11 12}
+do_execsql_test 2.1 {
+  SELECT * FROM t2 WHERE b=10;
+} {9 10 11 12}
+do_execsql_test 2.2 {
+  SELECT * FROM t2 WHERE c=11;
+} {9 10 11 12}
+do_execsql_test 2.3 {
+  SELECT * FROM t2 WHERE d=12;
+} {}
+do_execsql_test 2.4 {
+  SELECT * FROM t2 WHERE d='12';
+} {9 10 11 12}
+do_execsql_test 2.5 {
+  SELECT * FROM t2 WHERE a='9';
+} {9 10 11 12}
+
+do_execsql_test 3.0 {
+  DROP TABLE t2;
+  CREATE VIRTUAL TABLE temp.t3 USING csv(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4,
+    schema=
+      'CREATE TABLE t3(a PRIMARY KEY,b TEXT,c TEXT,d TEXT) WITHOUT ROWID',
+    testflags=1
+  );
+  SELECT a FROM t3 WHERE b=6 OR c=7 OR d=12 ORDER BY +a;
+} {5 9}
+do_execsql_test 3.1 {
+  SELECT a FROM t3 WHERE +b=6 OR c=7 OR d=12 ORDER BY +a;
+} {5 9}
+
+# The rowid column is not visible on a WITHOUT ROWID virtual table
+do_catchsql_test 3.2 {
+  SELECT rowid, a FROM t3;
+} {1 {no such column: rowid}}
+
+do_catchsql_test 4.0 {
+  DROP TABLE t3;
+  CREATE VIRTUAL TABLE temp.t4 USING csv_wr(
+    data=
+'1,2,3,4
+5,6,7,8
+9,10,11,12
+13,14,15,16
+',
+    columns=4,
+    schema=
+      'CREATE TABLE t3(a PRIMARY KEY,b TEXT,c TEXT,d TEXT) WITHOUT ROWID',
+    testflags=1
+  );
+} {1 {vtable constructor failed: t4}}
+
+finish_test

test/ctime.test

@@ -196,9 +196,10 @@ set tc 1
 foreach opt $opts {
   do_test ctime-2.5.$tc {
     set N [ expr {$tc-1} ]
-    set ans1 [ catchsql {
+    set ans1 [catch {db one {
       SELECT sqlite_compileoption_get($N);
-    } ]
+    }} msg]
+    lappend ans1 $msg
     set ans2 [ catchsql {
       SELECT sqlite_compileoption_used($opt);
     } ]

test/cursorhint2.test

@@ -0,0 +1,180 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library. The
+# focus is on testing that cursor-hints are correct for queries
+# involving LEFT JOIN.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix cursorhint2
+
+ifcapable !cursorhints {
+  finish_test
+  return
+}
+
+proc extract_hints {sql} {
+
+  db eval "SELECT tbl_name, rootpage FROM sqlite_master where rootpage" {
+    set lookup($rootpage) $tbl_name
+  }
+
+  set ret [list]
+  db eval "EXPLAIN $sql" a {
+    switch -- $a(opcode) {
+      OpenRead {
+        set csr($a(p1)) $lookup($a(p2))
+      }
+      CursorHint { 
+        lappend ret $csr($a(p1)) $a(p4) 
+      }
+    }
+  }
+
+  set ret
+}
+
+proc do_extract_hints_test {tn sql ret} {
+  uplevel [list do_test $tn [list extract_hints $sql] [list {*}$ret]]
+}
+
+do_execsql_test 1.0 {
+  PRAGMA automatic_index = 0;
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+  CREATE TABLE t3(e, f);
+}
+
+do_extract_hints_test 1.1 {
+  SELECT * FROM t1 WHERE a=1;
+} {
+  t1 EQ(c0,1)
+}
+
+do_extract_hints_test 1.2 {
+  SELECT * FROM t1 CROSS JOIN t2 ON (a=c) WHERE d IS NULL;
+} {
+  t2 {AND(ISNULL(c1),EQ(r[1],c0))}
+}
+
+do_extract_hints_test 1.3 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c) WHERE d IS NULL;
+} {
+  t2 {EQ(r[2],c0)}
+}
+
+do_extract_hints_test 1.4 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND a=10) WHERE d IS NULL;
+} {
+  t2 {AND(EQ(r[2],c0),EQ(r[3],10))}
+}
+
+do_extract_hints_test 1.5 {
+  SELECT * FROM t1 CROSS JOIN t2 ON (a=c AND a=10) WHERE d IS NULL;
+} {
+  t1 EQ(c0,10) t2 {AND(ISNULL(c1),EQ(r[3],c0))}
+}
+
+do_extract_hints_test 1.6 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c) LEFT JOIN t3 ON (d=f);
+} {
+  t2 {EQ(r[2],c0)} t3 {EQ(r[6],c1)}
+}
+
+do_extract_hints_test 1.7 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND d=e) LEFT JOIN t3 ON (d=f);
+} {
+  t2 {EQ(r[2],c0)} t3 {AND(EQ(r[6],c0),EQ(r[7],c1))}
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+  CREATE TABLE x1(x, y);
+  CREATE TABLE x2(a, b);
+}
+
+do_extract_hints_test 2.1 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NULL;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.2 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS +NULL;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.3 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = (b IS NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.4 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = coalesce(b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.5 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = coalesce(b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.6 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.7 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT +NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.8 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NOT +NULL
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.9 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE CASE b WHEN 0 THEN 0 ELSE 1 END;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.10 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b = 32+32
+} {
+  x2 {AND(EQ(c1,ADD(32,32)),EQ(c0,r[2]))}
+}
+
+do_extract_hints_test 2.11 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b LIKE 'abc%'
+} {
+  x2 {AND(expr,EQ(c0,r[2]))}
+}
+
+do_extract_hints_test 2.11 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE coalesce(x2.b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+finish_test
+

test/index6.test

@@ -376,4 +376,15 @@ do_execsql_test index6-10.3eqp {
   SELECT e FROM t10 WHERE a=1 AND b=2 ORDER BY d DESC;
 } {~/USING INDEX t10x/}
 
+# A partial index will be used for a full table scan, where possible
+do_execsql_test index6-11.1 {
+  CREATE TABLE t11(a,b,c);
+  CREATE INDEX t11x ON t11(a) WHERE b<>99;
+  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
+} {/USING INDEX t11x/}
+do_execsql_test index6-11.2 {
+  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
+} {/USING INDEX t11x/}
+  
+
 finish_test

test/selectA.test

@@ -1434,5 +1434,14 @@ do_catchsql_test 5.4 {
   SELECT * FROM t8 UNION SELECT * FROM t9 ORDER BY a+b COLLATE NOCASE
 } {1 {1st ORDER BY term does not match any column in the result set}}
 
+do_execsql_test 6.1 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(a INTEGER);
+  CREATE TABLE t2(b TEXT);
+  INSERT INTO t2(b) VALUES('12345');
+  SELECT * FROM (SELECT a FROM t1 UNION SELECT b FROM t2) WHERE a=a;
+} {12345}
+
 
 finish_test

test/speedtest1.c

@@ -1216,6 +1216,10 @@ static void displayLinuxIoStats(FILE *out){
 }   
 #endif
 
+#if SQLITE_VERSION_NUMBER<3006018
+#  define sqlite3_sourceid(X) "(before 3.6.18)"
+#endif
+
 int main(int argc, char **argv){
   int doAutovac = 0;            /* True for --autovacuum */
   int cacheSize = 0;            /* Desired cache size.  0 means default */

test/stat.test

@@ -32,6 +32,9 @@ db func a_string a_string
 
 register_dbstat_vtab db
 do_execsql_test stat-0.0 {
+  PRAGMA table_info(dbstat);
+} {/0 name TEXT .* 1 path TEXT .* 9 pgsize INTEGER/}
+do_execsql_test stat-0.1 {
   PRAGMA auto_vacuum = OFF;
   CREATE VIRTUAL TABLE temp.stat USING dbstat;
   SELECT * FROM stat;

test/tabfunc01.test

@@ -22,6 +22,7 @@ ifcapable !vtab {
   return
 }
 load_static_extension db series
+load_static_extension db carray
 
 do_execsql_test tabfunc01-1.1 {
   SELECT *, '|' FROM generate_series WHERE start=1 AND stop=9 AND step=2;
@@ -135,4 +136,61 @@ do_execsql_test tabfunc01-500 {
   ORDER BY +1;
 } {1 7 11 17}
 
+# Table-valued functions on the RHS of an IN operator
+#
+do_execsql_test tabfunc01-600 {
+  CREATE TABLE t600(a INTEGER PRIMARY KEY, b TEXT);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+    INSERT INTO t600(a,b) SELECT x, printf('(%03d)',x) FROM c;
+  SELECT b FROM t600 WHERE a IN generate_series(2,52,10);
+} {(002) (012) (022) (032) (042) (052)}
+
+
+do_test tabfunc01-700 {
+  set PTR [intarray_addr 5 7 13 17 23]
+  db eval {
+    SELECT b FROM t600, carray($PTR,5) WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+do_test tabfunc01-701 {
+  db eval {
+    SELECT b FROM t600 WHERE a IN carray($PTR,5,'int32');
+  }
+} {(005) (007) (013) (017) (023)}
+do_test tabfunc01-702 {
+  db eval {
+    SELECT b FROM t600 WHERE a IN carray($PTR,4,'int32');
+  }
+} {(005) (007) (013) (017)}
+do_catchsql_test tabfunc01-710 {
+  SELECT b FROM t600 WHERE a IN carray($PTR,5,'int33');
+} {1 {unknown datatype: 'int33'}}
+
+do_test tabfunc01-720 {
+  set PTR [int64array_addr 5 7 13 17 23]
+  db eval {
+    SELECT b FROM t600, carray($PTR,5,'int64') WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+
+do_test tabfunc01-730 {
+  set PTR [doublearray_addr 5.0 7.0 13.0 17.0 23.0]
+  db eval {
+    SELECT b FROM t600, carray($PTR,5,'double') WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+
+do_test tabfunc01-740 {
+  set PTR [textarray_addr 5 7 13 17 23]
+  db eval {
+    SELECT b FROM t600, carray($PTR,5,'char*') WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+
+
+intarray_addr
+int64array_addr
+doublearray_addr
+textarray_addr
+
 finish_test

test/tclsqlite.test

@@ -635,6 +635,39 @@ do_test tcl-14.2 {
   db one {SELECT x FROM t6 WHERE xCall()!='value'}
 } {}
 
+# Verify that the "exists" and "onecolumn" methods work when
+# a "profile" is registered.
+#
+catch {db close}
+sqlite3 db :memory:
+proc noop-profile {args} {
+  return
+}
+do_test tcl-15.0 {
+  db eval {CREATE TABLE t1(a); INSERT INTO t1 VALUES(1),(2),(3);}
+  db onecolumn {SELECT a FROM t1 WHERE a>2}
+} {3}
+do_test tcl-15.1 {
+  db exists {SELECT a FROM t1 WHERE a>2}
+} {1}
+do_test tcl-15.2 {
+  db exists {SELECT a FROM t1 WHERE a>3}
+} {0}
+db profile noop-profile
+do_test tcl-15.3 {
+  db onecolumn {SELECT a FROM t1 WHERE a>2}
+} {3}
+do_test tcl-15.4 {
+  db exists {SELECT a FROM t1 WHERE a>2}
+} {1}
+do_test tcl-15.5 {
+  db exists {SELECT a FROM t1 WHERE a>3}
+} {0}
+
+
+
+
+
 
 
 finish_test

test/walcrash4.test

@@ -35,6 +35,14 @@ do_execsql_test 1.0 {
 
 faultsim_save_and_close
 
+# The error message is different on unix and windows
+#
+if {$::tcl_platform(platform)=="windows"} {
+ set msg "child killed: unknown signal"
+} else {
+ set msg "child process exited abnormally"
+}
+
 for {set nExtra 0} {$nExtra < 10} {incr nExtra} {
   for {set i 0} {$i < 10} {incr i} {
     do_test 1.nExtra=$nExtra.i=$i.1 {
@@ -63,7 +71,7 @@ for {set nExtra 0} {$nExtra < 10} {incr nExtra} {
     
       set r [catch { exec [info nameofexec] crash.tcl >@stdout } msg]
       list $r $msg
-    } {1 {child process exited abnormally}}
+    } "1 {$msg}"
   
     do_execsql_test 1.nExtra=$nExtra.i=$i.2 { 
       SELECT count(*) FROM t1;

test/where3.test

@@ -45,7 +45,7 @@ do_test where3-1.1 {
   }
 } {222 two 2 222 {} {}}
 
-ifcapable explain {
+ifcapable explain&&!cursorhints {
   do_test where3-1.1.1 {
      explain_no_trace {SELECT * FROM t1, t2 LEFT JOIN t3 ON q=x
                         WHERE p=2 AND a=q}
@@ -86,7 +86,7 @@ do_test where3-1.2 {
   }
 } {1 {Value for C1.1} {Value for C2.1} 2 {} {Value for C2.2} 3 {Value for C1.3} {Value for C2.3}}
 
-ifcapable explain {
+ifcapable explain&&!cursorhints {
   do_test where3-1.2.1 {
      explain_no_trace {
        SELECT parent1.parent1key, child1.value, child2.value

tool/dbhash.c

@@ -0,0 +1,506 @@
+/*
+** 2016-06-07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This is a utility program that computes an SHA1 hash on the content
+** of an SQLite database.
+**
+** The hash is computed over just the content of the database.  Free
+** space inside of the database file, and alternative on-disk representations
+** of the same content (ex: UTF8 vs UTF16) do not affect the hash.  So,
+** for example, the database file page size, encoding, and auto_vacuum setting
+** can all be changed without changing the hash.
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <string.h>
+#include <assert.h>
+#include "sqlite3.h"
+
+/* Context for the SHA1 hash */
+typedef struct SHA1Context SHA1Context;
+struct SHA1Context {
+  unsigned int state[5];
+  unsigned int count[2];
+  unsigned char buffer[64];
+};
+
+/*
+** All global variables are gathered into the "g" singleton.
+*/
+struct GlobalVars {
+  const char *zArgv0;       /* Name of program */
+  unsigned fDebug;          /* Debug flags */
+  sqlite3 *db;              /* The database connection */
+  SHA1Context cx;           /* SHA1 hash context */
+} g;
+
+/*
+** Debugging flags
+*/
+#define DEBUG_FULLTRACE   0x00000001   /* Trace hash to stderr */
+
+/******************************************************************************
+** The Hash Engine
+**
+** Modify these routines (and appropriate state fields in global variable 'g')
+** in order to compute a different (better?) hash of the database.
+*/
+/*
+ * blk0() and blk() perform the initial expand.
+ * I got the idea of expanding during the round function from SSLeay
+ *
+ * blk0le() for little-endian and blk0be() for big-endian.
+ */
+#if __GNUC__ && (defined(__i386__) || defined(__x86_64__))
+/*
+ * GCC by itself only generates left rotates.  Use right rotates if
+ * possible to be kinder to dinky implementations with iterative rotate
+ * instructions.
+ */
+#define SHA_ROT(op, x, k) \
+        ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; })
+#define rol(x,k) SHA_ROT("roll", x, k)
+#define ror(x,k) SHA_ROT("rorl", x, k)
+
+#else
+/* Generic C equivalent */
+#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r))
+#define rol(x,k) SHA_ROT(x,k,32-(k))
+#define ror(x,k) SHA_ROT(x,32-(k),k)
+#endif
+
+
+#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \
+    |(rol(block[i],8)&0x00FF00FF))
+#define blk0be(i) block[i]
+#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \
+    ^block[(i+2)&15]^block[i&15],1))
+
+/*
+ * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1
+ *
+ * Rl0() for little-endian and Rb0() for big-endian.  Endianness is
+ * determined at run-time.
+ */
+#define Rl0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define Rb0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R1(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R2(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2);
+#define R3(v,w,x,y,z,i) \
+    z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2);
+#define R4(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2);
+
+/*
+ * Hash a single 512-bit block. This is the core of the algorithm.
+ */
+#define a qq[0]
+#define b qq[1]
+#define c qq[2]
+#define d qq[3]
+#define e qq[4]
+
+void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){
+  unsigned int qq[5]; /* a, b, c, d, e; */
+  static int one = 1;
+  unsigned int block[16];
+  memcpy(block, buffer, 64);
+  memcpy(qq,state,5*sizeof(unsigned int));
+
+  /* Copy g.cx.state[] to working vars */
+  /*
+  a = state[0];
+  b = state[1];
+  c = state[2];
+  d = state[3];
+  e = state[4];
+  */
+
+  /* 4 rounds of 20 operations each. Loop unrolled. */
+  if( 1 == *(unsigned char*)&one ){
+    Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3);
+    Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7);
+    Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11);
+    Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15);
+  }else{
+    Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3);
+    Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7);
+    Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11);
+    Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15);
+  }
+  R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19);
+  R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23);
+  R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27);
+  R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31);
+  R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35);
+  R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39);
+  R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43);
+  R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47);
+  R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51);
+  R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55);
+  R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59);
+  R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63);
+  R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67);
+  R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71);
+  R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75);
+  R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79);
+
+  /* Add the working vars back into context.state[] */
+  state[0] += a;
+  state[1] += b;
+  state[2] += c;
+  state[3] += d;
+  state[4] += e;
+}
+
+
+/* Initialize the SHA1 hash */
+static void hash_init(void){
+  /* SHA1 initialization constants */
+  g.cx.state[0] = 0x67452301;
+  g.cx.state[1] = 0xEFCDAB89;
+  g.cx.state[2] = 0x98BADCFE;
+  g.cx.state[3] = 0x10325476;
+  g.cx.state[4] = 0xC3D2E1F0;
+  g.cx.count[0] = g.cx.count[1] = 0;
+}
+
+/* Add new content to the SHA1 hash */
+static void hash_step(const unsigned char *data,  unsigned int len){
+  unsigned int i, j;
+
+  j = g.cx.count[0];
+  if( (g.cx.count[0] += len << 3) < j ){
+    g.cx.count[1] += (len>>29)+1;
+  }
+  j = (j >> 3) & 63;
+  if( (j + len) > 63 ){
+    (void)memcpy(&g.cx.buffer[j], data, (i = 64-j));
+    SHA1Transform(g.cx.state, g.cx.buffer);
+    for(; i + 63 < len; i += 64){
+      SHA1Transform(g.cx.state, &data[i]);
+    }
+    j = 0;
+  }else{
+    i = 0;
+  }
+  (void)memcpy(&g.cx.buffer[j], &data[i], len - i);
+}
+
+
+/* Add padding and compute and output the message digest. */
+static void hash_finish(const char *zName){
+  unsigned int i;
+  unsigned char finalcount[8];
+  unsigned char digest[20];
+  static const char zEncode[] = "0123456789abcdef";
+  char zOut[41];
+
+  for (i = 0; i < 8; i++){
+    finalcount[i] = (unsigned char)((g.cx.count[(i >= 4 ? 0 : 1)]
+       >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */
+  }
+  hash_step((const unsigned char *)"\200", 1);
+  while ((g.cx.count[0] & 504) != 448){
+    hash_step((const unsigned char *)"\0", 1);
+  }
+  hash_step(finalcount, 8);  /* Should cause a SHA1Transform() */
+  for (i = 0; i < 20; i++){
+    digest[i] = (unsigned char)((g.cx.state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
+  }
+  for(i=0; i<20; i++){
+    zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
+    zOut[i*2+1] = zEncode[digest[i] & 0xf];
+  }
+  zOut[i*2]= 0;
+  printf("%s %s\n", zOut, zName);
+}
+/* End of the hashing logic
+*******************************************************************************/
+  
+/*
+** Print an error resulting from faulting command-line arguments and
+** abort the program.
+*/
+static void cmdlineError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0);
+  exit(1);
+}
+
+/*
+** Print an error message for an error that occurs at runtime, then
+** abort the program.
+*/
+static void runtimeError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n");
+  exit(1);
+}
+
+/*
+** Prepare a new SQL statement.  Print an error and abort if anything
+** goes wrong.
+*/
+static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){
+  char *zSql;
+  int rc;
+  sqlite3_stmt *pStmt;
+
+  zSql = sqlite3_vmprintf(zFormat, ap);
+  if( zSql==0 ) runtimeError("out of memory");
+  rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0);
+  if( rc ){
+    runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db),
+                 zSql);
+  }
+  sqlite3_free(zSql);
+  return pStmt;
+}
+static sqlite3_stmt *db_prepare(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  return pStmt;
+}
+
+/*
+** Compute the hash for all rows of the query formed from the printf-style
+** zFormat and its argument.
+*/
+static void hash_one_query(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;        /* The query defined by zFormat and "..." */
+  int nCol;                   /* Number of columns in the result set */
+  int i;                      /* Loop counter */
+
+  /* Prepare the query defined by zFormat and "..." */
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  nCol = sqlite3_column_count(pStmt);
+
+  /* Compute a hash over the result of the query */
+  while( SQLITE_ROW==sqlite3_step(pStmt) ){
+    for(i=0; i<nCol; i++){
+      switch( sqlite3_column_type(pStmt,i) ){
+        case SQLITE_NULL: {
+          hash_step((const unsigned char*)"0",1);
+          if( g.fDebug & DEBUG_FULLTRACE ) fprintf(stderr, "NULL\n");
+          break;
+        }
+        case SQLITE_INTEGER: {
+          sqlite3_uint64 u;
+          int j;
+          unsigned char x[8];
+          sqlite3_int64 v = sqlite3_column_int64(pStmt,i);
+          memcpy(&u, &v, 8);
+          for(j=7; j>=0; j--){
+            x[j] = u & 0xff;
+            u >>= 8;
+          }
+          hash_step((const unsigned char*)"1",1);
+          hash_step(x,8);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "INT %s\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_FLOAT: {
+          sqlite3_uint64 u;
+          int j;
+          unsigned char x[8];
+          double r = sqlite3_column_double(pStmt,i);
+          memcpy(&u, &r, 8);
+          for(j=7; j>=0; j--){
+            x[j] = u & 0xff;
+            u >>= 8;
+          }
+          hash_step((const unsigned char*)"2",1);
+          hash_step(x,8);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "FLOAT %s\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_TEXT: {
+          int n = sqlite3_column_bytes(pStmt, i);
+          const unsigned char *z = sqlite3_column_text(pStmt, i);
+          hash_step((const unsigned char*)"3", 1);
+          hash_step(z, n);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "TEXT '%s'\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_BLOB: {
+          int n = sqlite3_column_bytes(pStmt, i);
+          const unsigned char *z = sqlite3_column_blob(pStmt, i);
+          hash_step((const unsigned char*)"4", 1);
+          hash_step(z, n);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "BLOB (%d bytes)\n", n);
+          }
+          break;
+        }
+      }
+    }
+  }
+  sqlite3_finalize(pStmt);
+}
+
+
+/*
+** Print sketchy documentation for this utility program
+*/
+static void showHelp(void){
+  printf("Usage: %s [options] FILE ...\n", g.zArgv0);
+  printf(
+"Compute a SHA1 hash on the content of database FILE.  System tables such as\n"
+"sqlite_stat1, sqlite_stat4, and sqlite_sequence are omitted from the hash.\n"
+"Options:\n"
+"   --debug N           Set debugging flags to N (experts only)\n"
+"   --like PATTERN      Only hash tables whose name is LIKE the pattern\n"
+"   --schema-only       Only hash the schema - omit table content\n"
+"   --without-schema    Only hash table content - omit the schema\n"
+  );
+}
+
+int main(int argc, char **argv){
+  const char *zDb = 0;         /* Name of the database currently being hashed */
+  int i;                       /* Loop counter */
+  int rc;                      /* Subroutine return code */
+  char *zErrMsg;               /* Error message when opening database */
+  sqlite3_stmt *pStmt;         /* An SQLite query */
+  const char *zLike = 0;       /* LIKE pattern of tables to hash */
+  int omitSchema = 0;          /* True to compute hash on content only */
+  int omitContent = 0;         /* True to compute hash on schema only */
+  int nFile = 0;               /* Number of input filenames seen */
+
+  g.zArgv0 = argv[0];
+  sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
+  for(i=1; i<argc; i++){
+    const char *z = argv[i];
+    if( z[0]=='-' ){
+      z++;
+      if( z[0]=='-' ) z++;
+      if( strcmp(z,"debug")==0 ){
+        if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
+        g.fDebug = strtol(argv[++i], 0, 0);
+      }else
+      if( strcmp(z,"help")==0 ){
+        showHelp();
+        return 0;
+      }else
+      if( strcmp(z,"like")==0 ){
+        if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
+        if( zLike!=0 ) cmdlineError("only one --like allowed");
+        zLike = argv[++i];
+      }else
+      if( strcmp(z,"schema-only")==0 ){
+        omitContent = 1;
+      }else
+      if( strcmp(z,"without-schema")==0 ){
+        omitSchema = 1;
+      }else
+      {
+        cmdlineError("unknown option: %s", argv[i]);
+      }
+    }else{
+      nFile++;
+      if( nFile<i ) argv[nFile] = argv[i];
+    }
+  }
+  if( nFile==0 ){
+    cmdlineError("no input files specified - nothing to do");
+  }
+  if( omitSchema && omitContent ){
+    cmdlineError("only one of --without-schema and --omit-schema allowed");
+  }
+  if( zLike==0 ) zLike = "%";
+
+  for(i=1; i<=nFile; i++){
+    static const int openFlags = 
+       SQLITE_OPEN_READWRITE |     /* Read/write so hot journals can recover */
+       SQLITE_OPEN_URI
+    ;
+    zDb = argv[i];
+    rc = sqlite3_open_v2(zDb, &g.db, openFlags, 0);
+    if( rc ){
+      fprintf(stderr, "cannot open database file '%s'\n", zDb);
+      continue;
+    }
+    rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_master", 0, 0, &zErrMsg);
+    if( rc || zErrMsg ){
+      sqlite3_close(g.db);
+      g.db = 0;
+      fprintf(stderr, "'%s' is not a valid SQLite database\n", zDb);
+      continue;
+    }
+
+    /* Start the hash */
+    hash_init();
+  
+    /* Hash table content */
+    if( !omitContent ){
+      pStmt = db_prepare(
+        "SELECT name FROM sqlite_master\n"
+        " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+        "   AND name NOT LIKE 'sqlite_%%'\n"
+        "   AND name LIKE '%q'\n"
+        " ORDER BY name COLLATE nocase;\n",
+        zLike
+      );
+      while( SQLITE_ROW==sqlite3_step(pStmt) ){
+        /* We want rows of the table to be hashed in PRIMARY KEY order.
+        ** Technically, an ORDER BY clause is required to guarantee that
+        ** order.  However, though not guaranteed by the documentation, every
+        ** historical version of SQLite has always output rows in PRIMARY KEY
+        ** order when there is no WHERE or GROUP BY clause, so the ORDER BY
+        ** can be safely omitted. */
+        hash_one_query("SELECT * FROM \"%w\"", sqlite3_column_text(pStmt,0));
+      }
+      sqlite3_finalize(pStmt);
+    }
+  
+    /* Hash the database schema */
+    if( !omitSchema ){
+      hash_one_query(
+         "SELECT type, name, tbl_name, sql FROM sqlite_master\n"
+         " WHERE tbl_name LIKE '%q'\n"
+         " ORDER BY name COLLATE nocase;\n",
+         zLike
+      );
+    }
+  
+    /* Finish and output the hash and close the database connection. */
+    hash_finish(zDb);
+    sqlite3_close(g.db);
+  }
+  return 0;
+}

tool/lempar.c

@@ -339,6 +339,9 @@ void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){
       pParser->yystack = &pParser->yystk0;
       pParser->yystksz = 1;
     }
+#endif
+#ifndef YYNOERRORRECOVERY
+    pParser->yyerrcnt = -1;
 #endif
     pParser->yytos = pParser->yystack;
     pParser->yystack[0].stateno = 0;
@@ -598,6 +601,9 @@ static void yy_shift(
     }
   }
 #endif
+  if( yyNewState > YY_MAX_SHIFT ){
+    yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+  }
   yytos = yypParser->yytos;
   yytos->stateno = (YYACTIONTYPE)yyNewState;
   yytos->major = (YYCODETYPE)yyMajor;
@@ -810,9 +816,6 @@ void Parse(
   do{
     yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
     if( yyact <= YY_MAX_SHIFTREDUCE ){
-      if( yyact > YY_MAX_SHIFT ){
-        yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
-      }
       yy_shift(yypParser,yyact,yymajor,yyminor);
 #ifndef YYNOERRORRECOVERY
       yypParser->yyerrcnt--;

tool/sqldiff.c

@@ -33,6 +33,7 @@ struct GlobalVars {
   const char *zArgv0;       /* Name of program */
   int bSchemaOnly;          /* Only show schema differences */
   int bSchemaPK;            /* Use the schema-defined PK, not the true PK */
+  int bHandleVtab;          /* Handle fts3, fts4, fts5 and rtree vtabs */
   unsigned fDebug;          /* Debug flags */
   sqlite3 *db;              /* The database connection */
 } g;
@@ -1735,6 +1736,144 @@ end_changeset_one_table:
   sqlite3_free(zId);
 }
 
+/*
+** Extract the next SQL keyword or quoted string from buffer zIn and copy it
+** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
+** Return a pointer to the character within zIn immediately following 
+** the token or quoted string just extracted.
+*/
+const char *gobble_token(const char *zIn, char *zBuf, int nBuf){
+  const char *p = zIn;
+  char *pOut = zBuf;
+  char *pEnd = &pOut[nBuf-1];
+  char q = 0;                     /* quote character, if any */
+
+  if( p==0 ) return 0;
+  while( *p==' ' ) p++;
+  switch( *p ){
+    case '"': q = '"'; break;
+    case '\'': q = '\''; break;
+    case '`': q = '`'; break;
+    case '[': q = ']'; break;
+  }
+
+  if( q ){
+    p++;
+    while( *p && pOut<pEnd ){
+      if( *p==q ){
+        p++;
+        if( *p!=q ) break;
+      }
+      if( pOut<pEnd ) *pOut++ = *p;
+      p++;
+    }
+  }else{
+    while( *p && *p!=' ' && *p!='(' ){
+      if( pOut<pEnd ) *pOut++ = *p;
+      p++;
+    }
+  }
+
+  *pOut = '\0';
+  return p;
+}
+
+/*
+** This function is the implementation of SQL scalar function "module_name":
+**
+**   module_name(SQL)
+**
+** The only argument should be an SQL statement of the type that may appear
+** in the sqlite_master table. If the statement is a "CREATE VIRTUAL TABLE"
+** statement, then the value returned is the name of the module that it
+** uses. Otherwise, if the statement is not a CVT, NULL is returned.
+*/
+static void module_name_func(
+  sqlite3_context *pCtx, 
+  int nVal, sqlite3_value **apVal
+){
+  const char *zSql;
+  char zToken[32];
+
+  assert( nVal==1 );
+  zSql = (const char*)sqlite3_value_text(apVal[0]);
+
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "create") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "virtual") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "table") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "using") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  
+  sqlite3_result_text(pCtx, zToken, -1, SQLITE_TRANSIENT);
+}
+
+/*
+** Return the text of an SQL statement that itself returns the list of
+** tables to process within the database.
+*/
+const char *all_tables_sql(){
+  if( g.bHandleVtab ){
+    int rc;
+  
+    rc = sqlite3_exec(g.db, 
+        "CREATE TEMP TABLE tblmap(module, postfix);"
+        "INSERT INTO temp.tblmap VALUES"
+        "('fts3', '_content'), ('fts3', '_segments'), ('fts3', '_segdir'),"
+  
+        "('fts4', '_content'), ('fts4', '_segments'), ('fts4', '_segdir'),"
+        "('fts4', '_docsize'), ('fts4', '_stat'),"
+  
+        "('fts5', '_data'), ('fts5', '_idx'), ('fts5', '_content'),"
+        "('fts5', '_docsize'), ('fts5', '_config'),"
+  
+        "('rtree', '_node'), ('rtree', '_rowid'), ('rtree', '_parent');"
+        , 0, 0, 0
+    );
+    assert( rc==SQLITE_OK );
+  
+    rc = sqlite3_create_function(
+        g.db, "module_name", 1, SQLITE_UTF8, 0, module_name_func, 0, 0
+    );
+    assert( rc==SQLITE_OK );
+  
+    return 
+      "SELECT name FROM main.sqlite_master\n"
+      " WHERE type='table' AND (\n"
+      "    module_name(sql) IS NULL OR \n"
+      "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
+      " ) AND name NOT IN (\n"
+      "  SELECT a.name || b.postfix \n"
+        "FROM main.sqlite_master AS a, temp.tblmap AS b \n"
+        "WHERE module_name(a.sql) = b.module\n" 
+      " )\n"
+      "UNION \n"
+      "SELECT name FROM aux.sqlite_master\n"
+      " WHERE type='table' AND (\n"
+      "    module_name(sql) IS NULL OR \n"
+      "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
+      " ) AND name NOT IN (\n"
+      "  SELECT a.name || b.postfix \n"
+        "FROM aux.sqlite_master AS a, temp.tblmap AS b \n"
+        "WHERE module_name(a.sql) = b.module\n" 
+      " )\n"
+      " ORDER BY name";
+  }else{
+    return
+      "SELECT name FROM main.sqlite_master\n"
+      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+      " UNION\n"
+      "SELECT name FROM aux.sqlite_master\n"
+      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+      " ORDER BY name";
+  }
+}
+
 /*
 ** Print sketchy documentation for this utility program
 */
@@ -1751,6 +1890,7 @@ static void showHelp(void){
 "  --summary             Show only a summary of the differences\n"
 "  --table TAB           Show only differences in table TAB\n"
 "  --transaction         Show SQL output inside a transaction\n"
+"  --vtab                Handle fts3, fts4, fts5 and rtree tables\n"
   );
 }
 
@@ -1821,6 +1961,9 @@ int main(int argc, char **argv){
       if( strcmp(z,"transaction")==0 ){
         useTransaction = 1;
       }else
+      if( strcmp(z,"vtab")==0 ){
+        g.bHandleVtab = 1;
+      }else
       {
         cmdlineError("unknown option: %s", argv[i]);
       }
@@ -1875,14 +2018,7 @@ int main(int argc, char **argv){
     xDiff(zTab, out);
   }else{
     /* Handle tables one by one */
-    pStmt = db_prepare(
-      "SELECT name FROM main.sqlite_master\n"
-      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
-      " UNION\n"
-      "SELECT name FROM aux.sqlite_master\n"
-      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
-      " ORDER BY name"
-    );
+    pStmt = db_prepare( all_tables_sql() );
     while( SQLITE_ROW==sqlite3_step(pStmt) ){
       xDiff((const char*)sqlite3_column_text(pStmt,0), out);
     }