Update the omit-table-from-left-join optimization so that it can omit tables

from the middle of the join as well as the end. FossilOrigin-Name: 618ca9fe53d8d2d7b4f368e6ee404d5fceeecac0d689f32ab62af8a6cbb37401
Fix a problem preventing the planner from identifying scans that visit at most
2017-11-21 19:22:45 +00:00 · 2017-11-20 15:46:10 +00:00 · 2017-11-20 15:45:03 +00:00 · 2017-11-20 14:40:03 +00:00 · 2017-11-18 18:07:17 +00:00 · 2017-11-18 17:30:08 +00:00
658 changed files with 85927 additions and 13122 deletions
@@ -166,7 +166,8 @@ USE_AMALGAMATION = @USE_AMALGAMATION@
 #
 LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
         backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
-         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
+         callback.lo complete.lo ctime.lo \
+         date.lo dbpage.lo dbstat.lo delete.lo \
         expr.lo fault.lo fkey.lo \
         fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
         fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
@@ -181,7 +182,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
         notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
         pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
         random.lo resolve.lo rowset.lo rtree.lo \
-         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
+         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
         table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
         update.lo util.lo vacuum.lo \
         vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
@@ -215,6 +216,7 @@ SRC = \
  $(TOP)/src/complete.c \
  $(TOP)/src/ctime.c \
  $(TOP)/src/date.c \
+  $(TOP)/src/dbpage.c \
  $(TOP)/src/dbstat.c \
  $(TOP)/src/delete.c \
  $(TOP)/src/expr.c \
@@ -265,7 +267,7 @@ SRC = \
  $(TOP)/src/rowset.c \
  $(TOP)/src/select.c \
  $(TOP)/src/status.c \
-  $(TOP)/src/shell.c \
+  $(TOP)/src/shell.c.in \
  $(TOP)/src/sqlite.h.in \
  $(TOP)/src/sqlite3ext.h \
  $(TOP)/src/sqliteInt.h \
@@ -350,7 +352,8 @@ SRC += \
  $(TOP)/ext/rbu/sqlite3rbu.h \
  $(TOP)/ext/rbu/sqlite3rbu.c
 SRC += \
-  $(TOP)/ext/misc/json1.c
+  $(TOP)/ext/misc/json1.c \
+  $(TOP)/ext/misc/stmt.c

 # Generated source code files
 #
@@ -361,6 +364,7 @@ SRC += \
  parse.c \
  parse.h \
  config.h \
+  shell.c \
  sqlite3.h

 # Source code to the test files.
@@ -382,6 +386,7 @@ TESTSRC = \
  $(TOP)/src/test_blob.c \
  $(TOP)/src/test_btree.c \
  $(TOP)/src/test_config.c \
+  $(TOP)/src/test_delete.c \
  $(TOP)/src/test_demovfs.c \
  $(TOP)/src/test_devsym.c \
  $(TOP)/src/test_fs.c \
@@ -391,6 +396,7 @@ TESTSRC = \
  $(TOP)/src/test_intarray.c \
  $(TOP)/src/test_journal.c \
  $(TOP)/src/test_malloc.c \
+  $(TOP)/src/test_md5.c \
  $(TOP)/src/test_multiplex.c \
  $(TOP)/src/test_mutex.c \
  $(TOP)/src/test_onefile.c \
@@ -402,6 +408,7 @@ TESTSRC = \
  $(TOP)/src/test_server.c \
  $(TOP)/src/test_superlock.c \
  $(TOP)/src/test_syscall.c \
+  $(TOP)/src/test_tclsh.c \
  $(TOP)/src/test_tclvar.c \
  $(TOP)/src/test_thread.c \
  $(TOP)/src/test_vfs.c \
@@ -426,12 +433,15 @@ TESTSRC += \
  $(TOP)/ext/fts5/fts5_test_mi.c \
  $(TOP)/ext/fts5/fts5_test_tok.c \
  $(TOP)/ext/misc/ieee754.c \
+  $(TOP)/ext/misc/mmapwarm.c \
  $(TOP)/ext/misc/nextchar.c \
  $(TOP)/ext/misc/percentile.c \
  $(TOP)/ext/misc/regexp.c \
+  $(TOP)/ext/misc/remember.c \
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \
+  $(TOP)/ext/misc/unionvtab.c \
  $(TOP)/ext/misc/wholenumber.c

 # Source code to the library files needed by the test fixture
@@ -444,6 +454,7 @@ TESTSRC2 = \
  $(TOP)/src/build.c \
  $(TOP)/src/ctime.c \
  $(TOP)/src/date.c \
+  $(TOP)/src/dbpage.c \
  $(TOP)/src/dbstat.c \
  $(TOP)/src/expr.c \
  $(TOP)/src/func.c \
@@ -481,7 +492,8 @@ TESTSRC2 = \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_write.c \
  $(TOP)/ext/async/sqlite3async.c \
-  $(TOP)/ext/session/sqlite3session.c 
+  $(TOP)/ext/session/sqlite3session.c \
+  $(TOP)/ext/misc/stmt.c 

 # Header files used by all library source files.
 #
@@ -549,7 +561,8 @@ FUZZDATA = \
  $(TOP)/test/fuzzdata1.db \
  $(TOP)/test/fuzzdata2.db \
  $(TOP)/test/fuzzdata3.db \
-  $(TOP)/test/fuzzdata4.db
+  $(TOP)/test/fuzzdata4.db \
+  $(TOP)/test/fuzzdata5.db

 # Standard options to testfixture
 #
@@ -558,9 +571,17 @@ TESTOPTS = --verbose=file --output=test-out.txt
 # Extra compiler options for various shell tools
 #
 SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
+# SHELL_OPT += -DSQLITE_ENABLE_FTS5
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
+SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
+SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
+SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
+SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
+FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
+FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
+DBFUZZ_OPT = 

 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
@@ -584,9 +605,9 @@ libtclsqlite3.la:	tclsqlite.lo libsqlite3.la
 		-version-info "8:6:8" \
 		-avoid-version

-sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
+sqlite3$(TEXE):	shell.c sqlite3.c
 	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
-		$(TOP)/src/shell.c sqlite3.c \
+		shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"

 sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
@@ -609,8 +630,15 @@ fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
 	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)

-fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
-	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
+fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h
+	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
+
+ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
+	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
+             $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
+
+dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
+	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)

 mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
 	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
@@ -648,7 +676,7 @@ mptest:	mptester$(TEXE)

 sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
-	cp tsrc/shell.c tsrc/sqlite3ext.h .
+	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .

 sqlite3ext.h:	.target_source
@@ -674,6 +702,11 @@ lemon$(BEXE):	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
 	$(BCC) -o $@ $(TOP)/tool/lemon.c
 	cp $(TOP)/tool/lempar.c .

+# Rules to build the program that generates the source-id
+#
+mksourceid$(BEXE):	$(TOP)/tool/mksourceid.c
+	$(BCC) -o $@ $(TOP)/tool/mksourceid.c
+
 # Rules to build individual *.o files from generated *.c files. This
 # applies to:
 #
@@ -727,6 +760,9 @@ ctime.lo:	$(TOP)/src/ctime.c $(HDR)
 date.lo:	$(TOP)/src/date.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c

+dbpage.lo:	$(TOP)/src/dbpage.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbpage.c
+
 dbstat.lo:	$(TOP)/src/dbstat.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c

@@ -911,7 +947,7 @@ tclsqlite.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c

 tclsqlite-shell.lo:	$(TOP)/src/tclsqlite.c $(HDR)
-	$(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c
+	$(LTCOMPILE) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c

 tclsqlite-stubs.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
@@ -939,13 +975,24 @@ parse.c:	$(TOP)/src/parse.y lemon$(BEXE) $(TOP)/tool/addopcodes.tcl
 	mv parse.h parse.h.temp
 	$(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h

-sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
+sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h

 keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
 	./mkkeywordhash$(BEXE) >keywordhash.h

+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)/src/shell.c.in \
+	$(TOP)/ext/misc/shathree.c \
+	$(TOP)/ext/misc/fileio.c \
+	$(TOP)/ext/misc/completion.c
+
+shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
+	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
+
+


 # Rules to build the extension objects.
@@ -1019,6 +1066,9 @@ sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
 json1.lo:	$(TOP)/ext/misc/json1.c
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c

+stmt.lo:	$(TOP)/ext/misc/stmt.c
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c
+
 # FTS5 things
 #
 FTS5_SRC = \
@@ -1063,11 +1113,14 @@ sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 # necessary because the test fixture requires non-API symbols which are
 # hidden when the library is built via the amalgamation).
 #
-TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
 TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
+TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
+TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB

 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
 TESTFIXTURE_SRC1 = sqlite3.c
@@ -1100,6 +1153,11 @@ fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
 valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
 	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)

+# The veryquick.test TCL tests.
+#
+tcltest:	./testfixture$(TEXE)
+	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
+
 # Minimal testing that runs in less than 3 minutes
 #
 quicktest:	./testfixture$(TEXE)
@@ -1108,8 +1166,7 @@ quicktest:	./testfixture$(TEXE)
 # This is the common case.  Run many tests that do not take too long,
 # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
 #
-test:	$(TESTPROGS) sourcetest fastfuzztest
-	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
+test:	fastfuzztest sourcetest $(TESTPROGS) tcltest

 # Run a test using valgrind.  This can take a really long time
 # because valgrind is so much slower than a native machine.
@@ -1124,18 +1181,32 @@ valgrindtest:	$(TESTPROGS) valgrindfuzz
 smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)

-sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
-	echo "#define TCLSH 2" > $@
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
-	echo "; return zMainloop; }" >> $@
+sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c

 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)

+CHECKER_DEPS =\
+  $(TOP)/tool/mkccode.tcl \
+  sqlite3.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/ext/repair/sqlite3_checker.tcl \
+  $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/repair/checkfreelist.c \
+  $(TOP)/ext/misc/btreeinfo.c \
+  $(TOP)/ext/repair/sqlite3_checker.c.in
+
+sqlite3_checker.c:	$(CHECKER_DEPS)
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
+
+sqlite3_checker$(TEXE):	sqlite3_checker.c
+	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
+
+dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
+	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
+           $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
+
 showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)

@@ -1148,6 +1219,9 @@ showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
 showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)

+showshm$(TEXE):	$(TOP)/tool/showshm.c
+	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
+
 changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)

@@ -1160,8 +1234,13 @@ LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)

-speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
-	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
+speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.c
+	$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)
+
+KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
+
+kvtest$(TEXE):	$(TOP)/test/kvtest.c sqlite3.c
+	$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)

 rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
 	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
@@ -21,7 +21,28 @@ USE_AMALGAMATION = 1
 # Set this non-0 to enable full warnings (-W4, etc) when compiling.
 #
 !IFNDEF USE_FULLWARN
-USE_FULLWARN = 0
+USE_FULLWARN = 1
+!ENDIF
+
+# Set this non-0 to enable treating warnings as errors (-WX, etc) when
+# compiling.
+#
+!IFNDEF USE_FATAL_WARN
+USE_FATAL_WARN = 0
+!ENDIF
+
+# Set this non-0 to enable full runtime error checks (-RTC1, etc).  This
+# has no effect if (any) optimizations are enabled.
+#
+!IFNDEF USE_RUNTIME_CHECKS
+USE_RUNTIME_CHECKS = 0
+!ENDIF
+
+# Set this non-0 to create a SQLite amalgamation file that excludes the
+# various built-in extensions.
+#
+!IFNDEF MINIMAL_AMALGAMATION
+MINIMAL_AMALGAMATION = 0
 !ENDIF

 # Set this non-0 to use "stdcall" calling convention for the core library
@@ -263,12 +284,39 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 !ENDIF
 !ENDIF

+# <<mark>>
+# These are the names of the customized Tcl header files used by various parts
+# of this makefile when the stdcall calling convention is in use.  It is not
+# used for any other purpose.
+#
+!IFNDEF SQLITETCLH
+SQLITETCLH = sqlite_tcl.h
+!ENDIF
+
+!IFNDEF SQLITETCLDECLSH
+SQLITETCLDECLSH = sqlite_tclDecls.h
+!ENDIF
+
+# These are the additional targets that the targets that integrate with the
+# Tcl library should depend on when compiling, etc.
+#
+!IFNDEF SQLITE_TCL_DEP
+!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
+SQLITE_TCL_DEP = $(SQLITETCLDECLSH) $(SQLITETCLH)
+!ELSE
+SQLITE_TCL_DEP =
+!ENDIF
+!ENDIF
+# <</mark>>
+
 # These are the "standard" SQLite compilation options used when compiling for
 # the Windows platform.
 #
 !IFNDEF OPT_FEATURE_FLAGS
+!IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
+!ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF

@@ -452,6 +500,12 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
 TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
 !ENDIF

+# Check if warnings should be treated as errors when compiling.
+#
+!IF $(USE_FATAL_WARN)!=0
+TCC = $(TCC) -WX
+!ENDIF
+
 TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise
 RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)

@@ -464,20 +518,32 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
 #
 !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
 !IF "$(PLATFORM)"=="x86"
-CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
-SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
+CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
+SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
+# <<mark>>
+TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl
+# <</mark>>
 !ELSE
 !IFNDEF PLATFORM
-CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
-SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
+CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
+SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
+# <<mark>>
+TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl
+# <</mark>>
 !ELSE
 CORE_CCONV_OPTS =
 SHELL_CCONV_OPTS =
+# <<mark>>
+TEST_CCONV_OPTS =
+# <</mark>>
 !ENDIF
 !ENDIF
 !ELSE
 CORE_CCONV_OPTS =
 SHELL_CCONV_OPTS =
+# <<mark>>
+TEST_CCONV_OPTS =
+# <</mark>>
 !ENDIF

 # These are additional compiler options used for the core library.
@@ -630,12 +696,35 @@ RCC = $(RCC) -I$(TOP)\ext\session
 # options are necessary in order to allow debugging symbols to
 # work correctly with Visual Studio when using the amalgamation.
 #
+!IFNDEF MKSQLITE3C_TOOL
+!IF $(MINIMAL_AMALGAMATION)!=0
+MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c-noext.tcl
+!ELSE
+MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c.tcl
+!ENDIF
+!ENDIF
+
 !IFNDEF MKSQLITE3C_ARGS
 !IF $(DEBUG)>1
 MKSQLITE3C_ARGS = --linemacros
 !ELSE
 MKSQLITE3C_ARGS =
 !ENDIF
+!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
+MKSQLITE3C_ARGS = $(MKSQLITE3C_ARGS) --useapicall
+!ENDIF
+!ENDIF
+
+# The mksqlite3h.tcl script accepts some options on the command line.
+# When compiling with stdcall support, some of these options are
+# necessary.
+#
+!IFNDEF MKSQLITE3H_ARGS
+!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
+MKSQLITE3H_ARGS = --useapicall
+!ELSE
+MKSQLITE3H_ARGS =
+!ENDIF
 !ENDIF
 # <</mark>>

@@ -657,6 +746,10 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
 !IF $(DEBUG)>2
 TCC = $(TCC) -DSQLITE_DEBUG=1
 RCC = $(RCC) -DSQLITE_DEBUG=1
+!IF $(DYNAMIC_SHELL)==0
+TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
+RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
+!ENDIF
 !ENDIF

 !IF $(DEBUG)>4 || $(OSTRACE)!=0
@@ -718,11 +811,11 @@ TCLLIBDIR = c:\tcl\lib
 !ENDIF

 !IFNDEF LIBTCL
-LIBTCL = tcl85.lib
+LIBTCL = tcl86.lib
 !ENDIF

 !IFNDEF LIBTCLSTUB
-LIBTCLSTUB = tclstub85.lib
+LIBTCLSTUB = tclstub86.lib
 !ENDIF

 !IFNDEF LIBTCLPATH
@@ -752,7 +845,7 @@ LIBICU = icuuc.lib icuin.lib
 # specific Tcl shell to use.
 #
 !IFNDEF TCLSH_CMD
-TCLSH_CMD = tclsh85
+TCLSH_CMD = tclsh
 !ENDIF
 # <</mark>>

@@ -835,6 +928,10 @@ RCC = $(RCC) -D_DEBUG
 !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
 TCC = $(TCC) -Od
 BCC = $(BCC) -Od
+!IF $(USE_RUNTIME_CHECKS)!=0
+TCC = $(TCC) -RTC1
+BCC = $(BCC) -RTC1
+!ENDIF
 !ELSEIF $(OPTIMIZATIONS)>=3
 TCC = $(TCC) -Ox
 BCC = $(BCC) -Ox
@@ -994,7 +1091,8 @@ LTLIBS = $(LTLIBS) $(LIBICU)
 #
 LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
         backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
-         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
+         callback.lo complete.lo ctime.lo \
+         date.lo dbpage.lo dbstat.lo delete.lo \
         expr.lo fault.lo fkey.lo \
         fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
         fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
@@ -1057,6 +1155,7 @@ SRC00 = \
  $(TOP)\src\complete.c \
  $(TOP)\src\ctime.c \
  $(TOP)\src\date.c \
+  $(TOP)\src\dbpage.c \
  $(TOP)\src\dbstat.c \
  $(TOP)\src\delete.c \
  $(TOP)\src\expr.c \
@@ -1123,11 +1222,6 @@ SRC01 = \
  $(TOP)\src\wherecode.c \
  $(TOP)\src\whereexpr.c

-# Shell source code files.
-#
-SRC02 = \
-  $(TOP)\src\shell.c
-
 # Core miscellaneous files.
 #
 SRC03 = \
@@ -1197,7 +1291,8 @@ SRC07 = \
  $(TOP)\ext\rtree\rtree.c \
  $(TOP)\ext\session\sqlite3session.c \
  $(TOP)\ext\rbu\sqlite3rbu.c \
-  $(TOP)\ext\misc\json1.c
+  $(TOP)\ext\misc\json1.c \
+  $(TOP)\ext\misc\stmt.c

 # Extension header files, part 1.
 #
@@ -1233,11 +1328,22 @@ SRC11 = \
  keywordhash.h \
  opcodes.h \
  parse.h \
+  shell.c \
  $(SQLITE3H)

+# Generated Tcl header files
+#
+!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
+SRC12 = \
+  $(SQLITETCLH) \
+  $(SQLITETCLDECLSH)
+!ELSE
+SRC12 =
+!ENDIF
+
 # All source code files.
 #
-SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
+SRC = $(SRC00) $(SRC01) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)

 # Source code to the test files.
 #
@@ -1258,6 +1364,7 @@ TESTSRC = \
  $(TOP)\src\test_blob.c \
  $(TOP)\src\test_btree.c \
  $(TOP)\src\test_config.c \
+  $(TOP)\src\test_delete.c \
  $(TOP)\src\test_demovfs.c \
  $(TOP)\src\test_devsym.c \
  $(TOP)\src\test_fs.c \
@@ -1267,6 +1374,7 @@ TESTSRC = \
  $(TOP)\src\test_intarray.c \
  $(TOP)\src\test_journal.c \
  $(TOP)\src\test_malloc.c \
+  $(TOP)\src\test_md5.c \
  $(TOP)\src\test_multiplex.c \
  $(TOP)\src\test_mutex.c \
  $(TOP)\src\test_onefile.c \
@@ -1278,6 +1386,7 @@ TESTSRC = \
  $(TOP)\src\test_server.c \
  $(TOP)\src\test_superlock.c \
  $(TOP)\src\test_syscall.c \
+  $(TOP)\src\test_tclsh.c \
  $(TOP)\src\test_tclvar.c \
  $(TOP)\src\test_thread.c \
  $(TOP)\src\test_vfs.c \
@@ -1302,12 +1411,15 @@ TESTEXT = \
  $(TOP)\ext\fts5\fts5_test_mi.c \
  $(TOP)\ext\fts5\fts5_test_tok.c \
  $(TOP)\ext\misc\ieee754.c \
+  $(TOP)\ext\misc\mmapwarm.c \
  $(TOP)\ext\misc\nextchar.c \
  $(TOP)\ext\misc\percentile.c \
  $(TOP)\ext\misc\regexp.c \
+  $(TOP)\ext\misc\remember.c \
  $(TOP)\ext\misc\series.c \
  $(TOP)\ext\misc\spellfix.c \
  $(TOP)\ext\misc\totype.c \
+  $(TOP)\ext\misc\unionvtab.c \
  $(TOP)\ext\misc\wholenumber.c

 # Source code to the library files needed by the test fixture
@@ -1341,7 +1453,7 @@ HDR = \
   parse.h \
   $(TOP)\src\pragma.h \
   $(SQLITE3H) \
-   $(TOP)\src\sqlite3ext.h \
+   sqlite3ext.h \
   $(TOP)\src\sqliteInt.h \
   $(TOP)\src\sqliteLimit.h \
   $(TOP)\src\vdbe.h \
@@ -1379,6 +1491,7 @@ TESTPROGS = \
  testfixture.exe \
  $(SQLITE3EXE) \
  sqlite3_analyzer.exe \
+  sqlite3_checker.exe \
  sqldiff.exe \
  dbhash.exe

@@ -1388,22 +1501,30 @@ FUZZDATA = \
  $(TOP)\test\fuzzdata1.db \
  $(TOP)\test\fuzzdata2.db \
  $(TOP)\test\fuzzdata3.db \
-  $(TOP)\test\fuzzdata4.db
+  $(TOP)\test\fuzzdata4.db \
+  $(TOP)\test\fuzzdata5.db
 # <</mark>>

 # Additional compiler options for the shell.  These are only effective
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
 !ENDIF

 # <<mark>>
 # Extra compiler options for various test tools.
 #
-MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
+MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
+FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
+OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
+DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
+KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
+DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
+ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0

 # Standard options to testfixture.
 #
@@ -1446,12 +1567,12 @@ $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
 sqlite3.def:	libsqlite3.lib
 	echo EXPORTS > sqlite3.def
 	dumpbin /all libsqlite3.lib \
-		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup)?_[^@]*)(?:@\d+)?$$" \1 \
 		| sort >> sqlite3.def
 # <</block2>>

-$(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
+$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)

 # <<mark>>
@@ -1473,8 +1594,14 @@ sourcetest:	srcck1.exe sqlite3.c
 fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-fuzzcheck.exe:	$(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+dbfuzz.exe:	$(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
+fuzzcheck.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
+ossshell.exe:	$(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

 mptester.exe:	$(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1499,12 +1626,11 @@ mptest:	mptester.exe
 # files are automatically generated.  This target takes care of
 # all that automatic generation.
 #
-.target_source:	$(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c
+.target_source:	$(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c $(SQLITE_TCL_DEP)
 	-rmdir /Q/S tsrc 2>NUL
 	-mkdir tsrc
 	for %i in ($(SRC00)) do copy /Y %i tsrc
 	for %i in ($(SRC01)) do copy /Y %i tsrc
-	for %i in ($(SRC02)) do copy /Y %i tsrc
 	for %i in ($(SRC03)) do copy /Y %i tsrc
 	for %i in ($(SRC04)) do copy /Y %i tsrc
 	for %i in ($(SRC05)) do copy /Y %i tsrc
@@ -1514,6 +1640,7 @@ mptest:	mptester.exe
 	for %i in ($(SRC09)) do copy /Y %i tsrc
 	for %i in ($(SRC10)) do copy /Y %i tsrc
 	for %i in ($(SRC11)) do copy /Y %i tsrc
+	for %i in ($(SRC12)) do copy /Y %i tsrc
 	copy /Y fts5.c tsrc
 	copy /Y fts5.h tsrc
 	del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL
@@ -1521,9 +1648,8 @@ mptest:	mptester.exe
 	move vdbe.new tsrc\vdbe.c
 	echo > .target_source

-sqlite3.c:	.target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl
-	$(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl $(MKSQLITE3C_ARGS)
-	copy tsrc\shell.c .
+sqlite3.c:	.target_source sqlite3ext.h $(MKSQLITE3C_TOOL)
+	$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS)
 	copy $(TOP)\ext\session\sqlite3session.h .

 sqlite3-all.c:	sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
@@ -1545,6 +1671,12 @@ lemon.exe:	$(TOP)\tool\lemon.c lempar.c
 	$(BCC) $(NO_WARN) -Daccess=_access \
 		-Fe$@ $(TOP)\tool\lemon.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)

+# <<mark>>
+# Rules to build the source-id generator tool
+#
+mksourceid.exe:	$(TOP)\tool\mksourceid.c
+	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\mksourceid.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
+
 # Rules to build individual *.lo files from generated *.c files. This
 # applies to:
 #
@@ -1615,7 +1747,10 @@ ctime.lo:	$(TOP)\src\ctime.c $(HDR)
 date.lo:	$(TOP)\src\date.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\date.c

-dbstat.lo:	$(TOP)\src\date.c $(HDR)
+dbpage.lo:	$(TOP)\src\dbpage.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbpage.c
+
+dbstat.lo:	$(TOP)\src\dbstat.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbstat.c

 delete.lo:	$(TOP)\src\delete.c $(HDR)
@@ -1795,11 +1930,11 @@ wherecode.lo:	$(TOP)\src\wherecode.c $(HDR)
 whereexpr.lo:	$(TOP)\src\whereexpr.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\whereexpr.c

-tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR)
+tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
 	$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c

-tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR)
-	$(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
+tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
+	$(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c

 tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
 	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
@@ -1823,11 +1958,17 @@ parse.c:	$(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl
 	move parse.h parse.h.temp
 	$(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h

-$(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
-	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H)
+$(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest mksourceid.exe $(TOP)\VERSION
+	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS)

 sqlite3ext.h:	.target_source
-	copy tsrc\sqlite3ext.h .
+!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
+	type tsrc\sqlite3ext.h | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*\)" "(SQLITE_CALLBACK *)" \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_APICALL *" > sqlite3ext.h
+	copy /Y sqlite3ext.h tsrc\sqlite3ext.h
+!ELSE
+	copy /Y tsrc\sqlite3ext.h sqlite3ext.h
+!ENDIF

 mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
 	$(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \
@@ -1836,6 +1977,15 @@ mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
 keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 	.\mkkeywordhash.exe > keywordhash.h

+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)\src\shell.c.in \
+	$(TOP)\ext\misc\shathree.c \
+	$(TOP)\ext\misc\fileio.c \
+	$(TOP)\ext\misc\completion.c
+
+shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
+	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c


 # Rules to build the extension objects.
@@ -1955,11 +2105,14 @@ sqlite3rbu.lo:	$(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR)
 # necessary because the test fixture requires non-API symbols which are
 # hidden when the library is built via the amalgamation).
 #
-TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS = -DTCLSH_INIT_PROC=sqlite3TestInit -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)

 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
 TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
@@ -1969,7 +2122,27 @@ TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1)
 !ENDIF

-testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR)
+!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
+sqlite_tclDecls.h:
+	echo #ifndef SQLITE_TCLAPI > $(SQLITETCLDECLSH)
+	echo #  define SQLITE_TCLAPI >> $(SQLITETCLDECLSH)
+	echo #endif >> $(SQLITETCLDECLSH)
+	type "$(TCLINCDIR)\tclDecls.h" \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN(?: CONST\d+?)?\s+?[^\(]*?\s+?)Tcl_" "\1 SQLITE_TCLAPI Tcl_" \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN\s+?(?:void|VOID)\s+?)TclFreeObj" "\1 SQLITE_TCLAPI TclFreeObj" \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tcl_" "(SQLITE_TCLAPI *tcl_" \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tclFreeObj" "(SQLITE_TCLAPI *tclFreeObj" \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_TCLAPI *" >> $(SQLITETCLDECLSH)
+
+sqlite_tcl.h:
+	type "$(TCLINCDIR)\tcl.h" | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact tclDecls.h sqlite_tclDecls.h \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "typedef (.*?)\(Tcl_" "typedef \1 (SQLITE_TCLAPI Tcl_" \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "void (*freeProc)" "void (SQLITE_TCLAPI *freeProc)" \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*findProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *findProc)" \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH)
+!ENDIF
+
+testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
 		-DBUILD_sqlite -I$(TCLINCDIR) \
 		$(TESTFIXTURE_SRC) \
@@ -2018,19 +2191,34 @@ smoketest:	$(TESTPROGS)
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)

-sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
-	echo #define TCLSH 2 > $@
-	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
-	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
-	echo static const char *tclsh_main_loop(void){ >> $@
-	echo static const char *zMainloop = >> $@
-	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@
-	echo ; return zMainloop; } >> $@
+sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP)
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@

 sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)

+CHECKER_DEPS =\
+  $(TOP)/tool/mkccode.tcl \
+  sqlite3.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/ext/repair/sqlite3_checker.tcl \
+  $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/repair/checkfreelist.c \
+  $(TOP)/ext/misc/btreeinfo.c \
+  $(TOP)/ext/repair/sqlite3_checker.c.in
+
+sqlite3_checker.c:	$(CHECKER_DEPS)
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@
+
+sqlite3_checker.exe:	sqlite3_checker.c $(LIBRESOBJS)
+	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \
+		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+
+dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
+		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
+
 testloadext.lo:	$(TOP)\src\test_loadext.c
 	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c

@@ -2038,48 +2226,62 @@ testloadext.dll:	testloadext.lo
 	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo

 showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

 showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

 showjournal.exe:	$(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

 showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C)
-	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
+showshm.exe:	$(TOP)\tool\showshm.c
+	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
+
+changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
 		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

 fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

 rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

 LogEst.exe:	$(TOP)\tool\logest.c $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
+	$(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)

 wordcount.exe:	$(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

 speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
+	$(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

+kvtest.exe:	$(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \
+		$(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
+dbselftest.exe:	$(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C)
+
 rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \
+	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \
 		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

+LSMDIR=$(TOP)\ext\lsm1
+!INCLUDE $(LSMDIR)\Makefile.msc
+
 moreclean:	clean
 	del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL
 # <</mark>>
@@ -2092,16 +2294,17 @@ clean:
 	del /Q sqlite3.c sqlite3.h 2>NUL
 	del /Q opcodes.c opcodes.h 2>NUL
 	del /Q lemon.* lempar.c parse.* 2>NUL
-	del /Q mkkeywordhash.* keywordhash.h 2>NUL
+	del /Q mksourceid.* mkkeywordhash.* keywordhash.h 2>NUL
 	del /Q notasharedlib.* 2>NUL
 	-rmdir /Q/S .deps 2>NUL
 	-rmdir /Q/S .libs 2>NUL
 	-rmdir /Q/S tsrc 2>NUL
 	del /Q .target_source 2>NUL
-	del /Q tclsqlite3.exe 2>NUL
+	del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL
+	del /Q lsm.dll lsmtest.exe 2>NUL
 	del /Q testloadext.dll 2>NUL
 	del /Q testfixture.exe test.db 2>NUL
-	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL
+	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL
 	del /Q changeset.exe 2>NUL
 	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
 	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
@@ -8,6 +8,50 @@ If you are reading this on a Git mirror someplace, you are doing it wrong.
 The [official repository](https://www.sqlite.org/src/) is better.  Go there
 now.

+## Obtaining The Code
+
+SQLite sources are managed using the
+[Fossil](https://www.fossil-scm.org/), a distributed version control system
+that was specifically designed to support SQLite development.
+If you do not want to use Fossil, you can download tarballs or ZIP
+archives as follows:
+
+  *  Lastest trunk check-in:
+     <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
+     <https://www.sqlite.org/src/zip/sqlite.zip>.
+
+  *  Latest release:
+     <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
+     <https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
+
+  *  For other check-ins, substitute an appropriate branch name or
+     tag or hash prefix for "release" in the URLs of the previous
+     bullet.  Or browse the [timeline](https://www.sqlite.org/src/timeline)
+     to locate the check-in desired, click on its information page link,
+     then click on the "Tarball" or "ZIP Archive" links on the information
+     page.
+
+If you do want to use Fossil to check out the source tree, 
+first install Fossil version 2.0 or later.
+(Source tarballs and precompiled binaries available
+[here](https://www.fossil-scm.org/fossil/uv/download.html).  Fossil is
+a stand-alone program.  To install, simply download or build the single 
+executable file and put that file someplace on your $PATH.)
+Then run commands like this:
+
+        mkdir ~/sqlite
+        cd ~/sqlite
+        fossil clone https://www.sqlite.org/src sqlite.fossil
+        fossil open sqlite.fossil
+    
+After setting up a repository using the steps above, you can always
+update to the lastest version using:
+
+        fossil update trunk   ;# latest trunk check-in
+        fossil update release ;# latest official release
+
+Or type "fossil ui" to get a web-based user interface.
+
 ## Compiling

 First create a directory in which to place
@@ -18,13 +62,13 @@ script found at the root of the source tree.  Then run "make".

 For example:

-    tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
-    mkdir bld                ;#  Build will occur in a sibling directory
-    cd bld                   ;#  Change to the build directory
-    ../sqlite/configure      ;#  Run the configure script
-    make                     ;#  Run the makefile.
-    make sqlite3.c           ;#  Build the "amalgamation" source file
-    make test                ;#  Run some tests (requires Tcl)
+        tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
+        mkdir bld                ;#  Build will occur in a sibling directory
+        cd bld                   ;#  Change to the build directory
+        ../sqlite/configure      ;#  Run the configure script
+        make                     ;#  Run the makefile.
+        make sqlite3.c           ;#  Build the "amalgamation" source file
+        make test                ;#  Run some tests (requires Tcl)

 See the makefile for additional targets.

@@ -43,13 +87,13 @@ with the provided "Makefile.msc" to build one of the supported targets.

 For example:

-    mkdir bld
-    cd bld
-    nmake /f Makefile.msc TOP=..\sqlite
-    nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
-    nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
-    nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
-    nmake /f Makefile.msc test TOP=..\sqlite
+        mkdir bld
+        cd bld
+        nmake /f Makefile.msc TOP=..\sqlite
+        nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
+        nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
+        nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
+        nmake /f Makefile.msc test TOP=..\sqlite

 There are several build options that can be set via the NMAKE command
 line.  For example, to build for WinRT, simply add "FOR_WINRT=1" argument
@@ -64,19 +108,22 @@ The makefiles also require AWK.

 ## Source Code Tour

-Most of the core source files are in the **src/** subdirectory.  But
-src/ also contains files used to build the "testfixture" test harness;
-those file all begin with "test".  And src/ contains the "shell.c" file
-which is the main program for the "sqlite3.exe" command-line shell and
-the "tclsqlite.c" file which implements the bindings to SQLite from the
-Tcl programming language.  (Historical note:  SQLite began as a Tcl
+Most of the core source files are in the **src/** subdirectory.  The
+**src/** folder also contains files used to build the "testfixture" test
+harness. The names of the source files used by "testfixture" all begin
+with "test".
+The **src/** also contains the "shell.c" file
+which is the main program for the "sqlite3.exe"
+[command-line shell](https://sqlite.org/cli.html) and
+the "tclsqlite.c" file which implements the
+[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite.
+(Historical note:  SQLite began as a Tcl
 extension and only later escaped to the wild as an independent library.)

 Test scripts and programs are found in the **test/** subdirectory.
-There are other test suites for SQLite (see
-[How SQLite Is Tested](http://www.sqlite.org/testing.html))
-but those other test suites are
-in separate source repositories.
+Addtional test code is found in other source repositories.
+See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
+additional information.

 The **ext/** subdirectory contains code for extensions.  The
 Full-text search engine is in **ext/fts3**.  The R-Tree engine is in
@@ -100,7 +147,7 @@ manually-edited files and automatically-generated files.
 The SQLite interface is defined by the **sqlite3.h** header file, which is
 generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
 [Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
-The manifest.uuid file contains the SHA1 hash of the particular check-in
+The manifest.uuid file contains the SHA3 hash of the particular check-in
 and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
 contains the current SQLite version number.  The sqlite3.h header is really
 just a copy of src/sqlite.h.in with the source-id and version number inserted
@@ -111,9 +158,8 @@ used to generate that documentation are in a separate source repository.
 The SQL language parser is **parse.c** which is generate from a grammar in
 the src/parse.y file.  The conversion of "parse.y" into "parse.c" is done
 by the [lemon](./doc/lemon.html) LALR(1) parser generator.  The source code
-for lemon is at tool/lemon.c.  Lemon uses a
-template for generating its parser.  A generic template is in tool/lempar.c,
-but SQLite uses a slightly modified template found in src/lempar.c.
+for lemon is at tool/lemon.c.  Lemon uses the tool/lempar.c file as a
+template for generating its parser.

 Lemon also generates the **parse.h** header file, at the same time it
 generates parse.c. But the parse.h header file is
@@ -133,6 +179,13 @@ that maps SQL language keywords (ex: "CREATE", "SELECT", "INDEX", etc.) into
 the numeric codes used by the parse.c parser.  The keywordhash.h file is
 generated by a C-language program at tool mkkeywordhash.c.

+The **pragma.h** header file contains various definitions used to parse
+and implement the PRAGMA statements.  The header is generated by a
+script **tool/mkpragmatab.tcl**. If you want to add a new PRAGMA, edit
+the **tool/mkpragmatab.tcl** file to insert the information needed by the
+parser for your new PRAGMA, then run the script to regenerate the
+**pragma.h** header file.
+
 ### The Amalgamation

 All of the individual C source code and header files (both manually-edited
@@ -150,7 +203,7 @@ subdirectory (using the equivalent of "make target_source") then the
 tool/mksqlite3c.tcl script is run to copy them all together in just the
 right order while resolving internal "#include" references.

-The amalgamation source file is more than 100K lines long.  Some symbolic
+The amalgamation source file is more than 200K lines long.  Some symbolic
 debuggers (most notably MSVC) are unable to deal with files longer than 64K
 lines.  To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
 can be run on the amalgamation to break it up into a single small C file
@@ -167,14 +220,15 @@ See the [architectural description](http://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
 (helping to understand how SQLite works include the
 [file format](http://www.sqlite.org/fileformat2.html) description,
-the [virtual machine](http://www.sqlite.org/vdbe.html) that runs
+the [virtual machine](http://www.sqlite.org/opcode.html) that runs
 prepared statements, the description of
 [how transactions work](http://www.sqlite.org/atomiccommit.html), and
 the [overview of the query planner](http://www.sqlite.org/optoverview.html).

-Unfortunately, years of effort have gone into optimizating SQLite, both
+Years of effort have gone into optimizating SQLite, both
 for small size and high performance.  And optimizations tend to result in
-complex code.  So there is a lot of complexity in the SQLite implementation.
+complex code.  So there is a lot of complexity in the current SQLite
+implementation.  It will not be the easiest library in the world to hack.

 Key files:

@@ -185,7 +239,7 @@ Key files:
  *  **sqliteInt.h** - this header file defines many of the data objects
     used internally by SQLite.

-  *  **parse.y** - This file describes the LALR(1) grammer that SQLite uses
+  *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
     to parse SQL statements, and the actions that are taken at each step
     in the parsing process.

@@ -218,13 +272,13 @@ Key files:
     is not part of the core SQLite library.  But as most of the tests in this
     repository are written in Tcl, the Tcl language bindings are important.

-There are many other source files.  Each has a suscinct header comment that
+There are many other source files.  Each has a succinct header comment that
 describes its purpose and role within the larger system.


 ## Contacts

 The main SQLite webpage is [http://www.sqlite.org/](http://www.sqlite.org/)
-with geographically distributed backup servers at
+with geographically distributed backups at
 [http://www2.sqlite.org/](http://www2.sqlite.org) and
 [http://www3.sqlite.org/](http://www3.sqlite.org).
@@ -1 +1 @@
-3.14.0
+3.22.0
@@ -10,7 +10,7 @@ sqlite3_SOURCES = shell.c sqlite3.h
 EXTRA_sqlite3_SOURCES = sqlite3.c
 sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
 sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
-sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
+sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB

 include_HEADERS = sqlite3.h sqlite3ext.h

@@ -21,7 +21,28 @@ TOP = .
 # Set this non-0 to enable full warnings (-W4, etc) when compiling.
 #
 !IFNDEF USE_FULLWARN
-USE_FULLWARN = 0
+USE_FULLWARN = 1
+!ENDIF
+
+# Set this non-0 to enable treating warnings as errors (-WX, etc) when
+# compiling.
+#
+!IFNDEF USE_FATAL_WARN
+USE_FATAL_WARN = 0
+!ENDIF
+
+# Set this non-0 to enable full runtime error checks (-RTC1, etc).  This
+# has no effect if (any) optimizations are enabled.
+#
+!IFNDEF USE_RUNTIME_CHECKS
+USE_RUNTIME_CHECKS = 0
+!ENDIF
+
+# Set this non-0 to create a SQLite amalgamation file that excludes the
+# various built-in extensions.
+#
+!IFNDEF MINIMAL_AMALGAMATION
+MINIMAL_AMALGAMATION = 0
 !ENDIF

 # Set this non-0 to use "stdcall" calling convention for the core library
@@ -248,12 +269,15 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 !ENDIF
 !ENDIF

+
 # These are the "standard" SQLite compilation options used when compiling for
 # the Windows platform.
 #
 !IFNDEF OPT_FEATURE_FLAGS
+!IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
+!ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF

@@ -437,6 +461,12 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
 TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
 !ENDIF

+# Check if warnings should be treated as errors when compiling.
+#
+!IF $(USE_FATAL_WARN)!=0
+TCC = $(TCC) -WX
+!ENDIF
+
 TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise
 RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)

@@ -449,12 +479,12 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)
 #
 !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
 !IF "$(PLATFORM)"=="x86"
-CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
-SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
+CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
+SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
 !ELSE
 !IFNDEF PLATFORM
-CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
-SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
+CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
+SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
 !ELSE
 CORE_CCONV_OPTS =
 SHELL_CCONV_OPTS =
@@ -615,6 +645,10 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
 !IF $(DEBUG)>2
 TCC = $(TCC) -DSQLITE_DEBUG=1
 RCC = $(RCC) -DSQLITE_DEBUG=1
+!IF $(DYNAMIC_SHELL)==0
+TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
+RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
+!ENDIF
 !ENDIF

 !IF $(DEBUG)>4 || $(OSTRACE)!=0
@@ -740,6 +774,10 @@ RCC = $(RCC) -D_DEBUG
 !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
 TCC = $(TCC) -Od
 BCC = $(BCC) -Od
+!IF $(USE_RUNTIME_CHECKS)!=0
+TCC = $(TCC) -RTC1
+BCC = $(BCC) -RTC1
+!ENDIF
 !ELSEIF $(OPTIMIZATIONS)>=3
 TCC = $(TCC) -Ox
 BCC = $(BCC) -Ox
@@ -889,7 +927,8 @@ LIBRESOBJS =
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
 !ENDIF


@@ -916,11 +955,11 @@ Replace.exe:
 sqlite3.def:	Replace.exe $(LIBOBJ)
 	echo EXPORTS > sqlite3.def
 	dumpbin /all $(LIBOBJ) \
-		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
+		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
 		| sort >> sqlite3.def

-$(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
+$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)


@@ -935,7 +974,7 @@ sqlite3.lo:	$(SQLITE3C)
 !IF $(USE_RC)!=0
 _HASHCHAR=^#
 !IF ![echo !IFNDEF VERSION > rcver.vc] && \
-    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
+    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| "%SystemRoot%\System32\find.exe" "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
    ![echo !ENDIF >> rcver.vc]
 !INCLUDE rcver.vc
 !ENDIF
@@ -12,6 +12,7 @@
 AC_PREREQ(2.61)
 AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org)
 AC_CONFIG_SRCDIR([sqlite3.c])
+AC_CONFIG_AUX_DIR([.])

 # Use automake.
 AM_INIT_AUTOMAKE([foreign])
@@ -55,9 +56,9 @@ AS_IF([ test x"$enable_editline" != xno ],[
    LIBS=""
    AC_SEARCH_LIBS([readline],[edit],[
      AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
-      READLINE_LIBS=$LIBS
+      READLINE_LIBS="$LIBS -ltinfo"
      enable_readline=no
-    ])
+    ],[],[-ltinfo])
    AS_UNSET(ac_cv_search_readline)
    LIBS=$sLIBS
  ])
@@ -78,7 +78,6 @@ TEA_ADD_LIBS([])
 TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1])
 TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1])
 TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1])
-TEA_ADD_CFLAGS([-DSQLITE_OMIT_DEPRECATED=1])
 TEA_ADD_STUB_SOURCES([])
 TEA_ADD_TCL_SOURCES([])

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.14.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.22.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.14.0'
-PACKAGE_STRING='sqlite 3.14.0'
+PACKAGE_VERSION='3.22.0'
+PACKAGE_STRING='sqlite 3.22.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -909,6 +909,7 @@ enable_fts3
 enable_fts4
 enable_fts5
 enable_json1
+enable_update_limit
 enable_rtree
 enable_session
 enable_gcov
@@ -1463,7 +1464,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.14.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.22.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1528,7 +1529,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.14.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.22.0:";;
   esac
  cat <<\_ACEOF

@@ -1560,6 +1561,7 @@ Optional Features:
  --enable-fts4           Enable the FTS4 extension
  --enable-fts5           Enable the FTS5 extension
  --enable-json1          Enable the JSON1 extension
+  --enable-update-limit   Enable the UPDATE/DELETE LIMIT clause
  --enable-rtree          Enable the RTREE extension
  --enable-session        Enable the SESSION extension
  --enable-gcov           Enable coverage testing using gcov
@@ -1652,7 +1654,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.14.0
+sqlite configure 3.22.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2071,7 +2073,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.14.0, which was
+It was created by sqlite $as_me 3.22.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -3929,13 +3931,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
  lt_cv_nm_interface="BSD nm"
  echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3932: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3934: $ac_compile\"" >&5)
  (eval "$ac_compile" 2>conftest.err)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3937: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3938: output\"" >&5)
+  (eval echo "\"\$as_me:3940: output\"" >&5)
  cat conftest.out >&5
  if $GREP 'External.*some_variable' conftest.out > /dev/null; then
    lt_cv_nm_interface="MS dumpbin"
@@ -5141,7 +5143,7 @@ ia64-*-hpux*)
  ;;
 *-*-irix6*)
  # Find out which ABI we are using.
-  echo '#line 5144 "configure"' > conftest.$ac_ext
+  echo '#line 5146 "configure"' > conftest.$ac_ext
  if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  (eval $ac_compile) 2>&5
  ac_status=$?
@@ -6666,11 +6668,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6669: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6671: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:6673: \$? = $ac_status" >&5
+   echo "$as_me:6675: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7005,11 +7007,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7008: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7010: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:7012: \$? = $ac_status" >&5
+   echo "$as_me:7014: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7110,11 +7112,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7113: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7115: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7117: \$? = $ac_status" >&5
+   echo "$as_me:7119: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -7165,11 +7167,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7168: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7170: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7172: \$? = $ac_status" >&5
+   echo "$as_me:7174: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -9545,7 +9547,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9548 "configure"
+#line 9550 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -9641,7 +9643,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9644 "configure"
+#line 9646 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -10302,7 +10304,7 @@ USE_AMALGAMATION=1
 # if not, then we fall back to plain tclsh.
 # TODO: try other versions before falling back?
 #
-for ac_prog in tclsh8.6 tclsh8.5 tclsh
+for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh
 do
  # Extract the first word of "$ac_prog", so it can be a program name with args.
 set dummy $ac_prog; ac_word=$2
@@ -11252,7 +11254,7 @@ else
 fi

 if test "${use_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1"
+  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 else
  TARGET_DEBUG="-DNDEBUG"
 fi
@@ -11356,7 +11358,7 @@ fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5
 $as_echo_n "checking whether to support MEMSYS5... " >&6; }
 if test "${enable_memsys5}" = "yes"; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 $as_echo "yes" >&6; }
 else
@@ -11373,7 +11375,7 @@ fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5
 $as_echo_n "checking whether to support MEMSYS3... " >&6; }
 if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 $as_echo "yes" >&6; }
 else
@@ -11391,7 +11393,7 @@ else
 fi

 if test "${enable_fts3}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
 fi
 # Check whether --enable-fts4 was given.
 if test "${enable_fts4+set}" = set; then :
@@ -11401,7 +11403,7 @@ else
 fi

 if test "${enable_fts4}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
 $as_echo_n "checking for library containing log... " >&6; }
 if ${ac_cv_search_log+:} false; then :
@@ -11467,7 +11469,7 @@ else
 fi

 if test "${enable_fts5}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
 $as_echo_n "checking for library containing log... " >&6; }
 if ${ac_cv_search_log+:} false; then :
@@ -11536,7 +11538,21 @@ else
 fi

 if test "${enable_json1}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
+fi
+
+#########
+# See whether we should enable the LIMIT clause on UPDATE and DELETE
+# statements.
+# Check whether --enable-update-limit was given.
+if test "${enable_update_limit+set}" = set; then :
+  enableval=$enable_update_limit; enable_udlimit=yes
+else
+  enable_udlimit=no
+fi
+
+if test "${enable_udlimit}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
 fi

 #########
@@ -11549,7 +11565,7 @@ else
 fi

 if test "${enable_rtree}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
 fi

 #########
@@ -11562,12 +11578,12 @@ else
 fi

 if test "${enable_session}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION"
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
 fi

 #########
-# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
+# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter
 for option in $CFLAGS $CPPFLAGS
 do
  case $option in
@@ -12151,7 +12167,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.14.0, which was
+This file was extended by sqlite $as_me 3.22.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12217,7 +12233,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.14.0
+sqlite config.status 3.22.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -120,7 +120,7 @@ USE_AMALGAMATION=1
 # if not, then we fall back to plain tclsh.
 # TODO: try other versions before falling back?
 # 
-AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh8.5 tclsh], none)
+AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none)
 if test "$TCLSH_CMD" = "none"; then
  # If we can't find a local tclsh, then building the amalgamation will fail.
  # We act as though --disable-amalgamation has been used.
@@ -560,7 +560,7 @@ AC_SEARCH_LIBS(fdatasync, [rt])
 AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
      [use_debug=$enableval],[use_debug=no])
 if test "${use_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1"
+  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 else
  TARGET_DEBUG="-DNDEBUG"
 fi
@@ -596,7 +596,7 @@ AC_ARG_ENABLE(memsys5,
  [enable_memsys5=yes],[enable_memsys5=no])
 AC_MSG_CHECKING([whether to support MEMSYS5])
 if test "${enable_memsys5}" = "yes"; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
  AC_MSG_RESULT([yes])
 else
  AC_MSG_RESULT([no])
@@ -606,7 +606,7 @@ AC_ARG_ENABLE(memsys3,
  [enable_memsys3=yes],[enable_memsys3=no])
 AC_MSG_CHECKING([whether to support MEMSYS3])
 if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
  AC_MSG_RESULT([yes])
 else
  AC_MSG_RESULT([no])
@@ -618,20 +618,20 @@ AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
      [Enable the FTS3 extension]),
      [enable_fts3=yes],[enable_fts3=no])
 if test "${enable_fts3}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
 fi
 AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
      [Enable the FTS4 extension]),
      [enable_fts4=yes],[enable_fts4=no])
 if test "${enable_fts4}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
  AC_SEARCH_LIBS([log],[m])
 fi
 AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
      [Enable the FTS5 extension]),
      [enable_fts5=yes],[enable_fts5=no])
 if test "${enable_fts5}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
  AC_SEARCH_LIBS([log],[m])
 fi

@@ -641,7 +641,17 @@ AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],
      [Enable the JSON1 extension]),
      [enable_json1=yes],[enable_json1=no])
 if test "${enable_json1}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
+fi
+
+#########
+# See whether we should enable the LIMIT clause on UPDATE and DELETE
+# statements.
+AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
+      [Enable the UPDATE/DELETE LIMIT clause]),
+      [enable_udlimit=yes],[enable_udlimit=no])
+if test "${enable_udlimit}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
 fi

 #########
@@ -650,7 +660,7 @@ AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
      [Enable the RTREE extension]),
      [enable_rtree=yes],[enable_rtree=no])
 if test "${enable_rtree}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
 fi

 #########
@@ -659,12 +669,12 @@ AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
      [Enable the SESSION extension]),
      [enable_session=yes],[enable_session=no])
 if test "${enable_session}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION"
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
 fi

 #########
-# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
+# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter
 for option in $CFLAGS $CPPFLAGS
 do
  case $option in
@@ -2,12 +2,12 @@
 <head>
 <title>The Lemon Parser Generator</title>
 </head>
-<body bgcolor=white>
-<h1 align=center>The Lemon Parser Generator</h1>
+<body bgcolor='white'>
+<h1 align='center'>The Lemon Parser Generator</h1>

 <p>Lemon is an LALR(1) parser generator for C.
 It does the same job as "bison" and "yacc".
-But lemon is not a bison or yacc clone.  Lemon
+But Lemon is not a bison or yacc clone.  Lemon
 uses a different grammar syntax which is designed to
 reduce the number of coding errors.  Lemon also uses a
 parsing engine that is faster than yacc and
@@ -16,13 +16,33 @@ bison and which is both reentrant and threadsafe.
 has also been updated so that it too can generate a
 reentrant and threadsafe parser.)
 Lemon also implements features that can be used
-to eliminate resource leaks, making is suitable for use
+to eliminate resource leaks, making it suitable for use
 in long-running programs such as graphical user interfaces
 or embedded controllers.</p>

 <p>This document is an introduction to the Lemon
 parser generator.</p>

+<h2>Security Note</h2>
+
+<p>The language parser code created by Lemon is very robust and
+is well-suited for use in internet-facing applications that need to
+safely process maliciously crafted inputs.
+
+<p>The "lemon.exe" command-line tool itself works great when given a valid
+input grammar file and almost always gives helpful
+error messages for malformed inputs.  However,  it is possible for
+a malicious user to craft a grammar file that will cause 
+lemon.exe to crash.
+We do not see this as a problem, as lemon.exe is not intended to be used
+with hostile inputs.
+To summarize:</p>
+
+<ul>
+<li>Parser code generated by lemon &rarr; Robust and secure
+<li>The "lemon.exe" command line tool itself &rarr; Not so much
+</ul>
+
 <h2>Theory of Operation</h2>

 <p>The main goal of Lemon is to translate a context free grammar (CFG)
@@ -38,8 +58,8 @@ Lemon comes with a default parser template which works fine for most
 applications.  But the user is free to substitute a different parser
 template if desired.</p>

-<p>Depending on command-line options, Lemon will generate between
-one and three files of outputs.
+<p>Depending on command-line options, Lemon will generate up to
+three output files.
 <ul>
 <li>C code to implement the parser.
 <li>A header file defining an integer ID for each terminal symbol.
@@ -70,27 +90,30 @@ the states used by the parser automaton.</p>
 You can obtain a list of the available command-line options together
 with a brief explanation of what each does by typing
 <pre>
-   lemon -?
+   lemon "-?"
 </pre>
 As of this writing, the following command-line options are supported:
 <ul>
 <li><b>-b</b>
 Show only the basis for each parser state in the report file.
 <li><b>-c</b>
-Do not compress the generated action tables.
+Do not compress the generated action tables.  The parser will be a
+little larger and slower, but it will detect syntax errors sooner.
 <li><b>-D<i>name</i></b>
-Define C preprocessor macro <i>name</i>.  This macro is useable by
-"%ifdef" lines in the grammar file.
+Define C preprocessor macro <i>name</i>.  This macro is usable by
+"<tt><a href='#pifdef'>%ifdef</a></tt>" and
+"<tt><a href='#pifdef'>%ifndef</a></tt>" lines
+in the grammar file.
 <li><b>-g</b>
 Do not generate a parser.  Instead write the input grammar to standard
 output with all comments, actions, and other extraneous text removed.
 <li><b>-l</b>
-Omit "#line" directives int the generated parser C code.
+Omit "#line" directives in the generated parser C code.
 <li><b>-m</b>
 Cause the output C source code to be compatible with the "makeheaders"
-program. 
+program.
 <li><b>-p</b>
-Display all conflicts that are resolved by 
+Display all conflicts that are resolved by
 <a href='#precrules'>precedence rules</a>.
 <li><b>-q</b>
 Suppress generation of the report file.
@@ -145,7 +168,7 @@ once for each token:
 </pre>
 The first argument to the Parse() routine is the pointer returned by
 ParseAlloc().
-The second argument is a small positive integer that tells the parse the
+The second argument is a small positive integer that tells the parser the
 type of the next token in the data stream.
 There is one token type for each terminal symbol in the grammar.
 The gram.h file generated by Lemon contains #define statements that
@@ -153,7 +176,7 @@ map symbolic terminal symbol names into appropriate integer values.
 A value of 0 for the second argument is a special flag to the
 parser to indicate that the end of input has been reached.
 The third argument is the value of the given token.  By default,
-the type of the third argument is integer, but the grammar will
+the type of the third argument is "void*", but the grammar will
 usually redefine this type to be some kind of structure.
 Typically the second argument will be a broad category of tokens
 such as "identifier" or "number" and the third argument will
@@ -161,7 +184,7 @@ be the name of the identifier or the value of the number.</p>

 <p>The Parse() function may have either three or four arguments,
 depending on the grammar.  If the grammar specification file requests
-it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>),
+it (via the <tt><a href='#extraarg'>%extra_argument</a></tt> directive),
 the Parse() function will have a fourth parameter that can be
 of any type chosen by the programmer.  The parser doesn't do anything
 with this argument except to pass it through to action routines.
@@ -171,20 +194,20 @@ to the action routines without having to use global variables.</p>
 <p>A typical use of a Lemon parser might look something like the
 following:
 <pre>
-   01 ParseTree *ParseFile(const char *zFilename){
-   02    Tokenizer *pTokenizer;
-   03    void *pParser;
-   04    Token sToken;
-   05    int hTokenId;
-   06    ParserState sState;
-   07
-   08    pTokenizer = TokenizerCreate(zFilename);
-   09    pParser = ParseAlloc( malloc );
-   10    InitParserState(&sState);
-   11    while( GetNextToken(pTokenizer, &hTokenId, &sToken) ){
-   12       Parse(pParser, hTokenId, sToken, &sState);
+    1 ParseTree *ParseFile(const char *zFilename){
+    2    Tokenizer *pTokenizer;
+    3    void *pParser;
+    4    Token sToken;
+    5    int hTokenId;
+    6    ParserState sState;
+    7
+    8    pTokenizer = TokenizerCreate(zFilename);
+    9    pParser = ParseAlloc( malloc );
+   10    InitParserState(&amp;sState);
+   11    while( GetNextToken(pTokenizer, &amp;hTokenId, &amp;sToken) ){
+   12       Parse(pParser, hTokenId, sToken, &amp;sState);
   13    }
-   14    Parse(pParser, 0, sToken, &sState);
+   14    Parse(pParser, 0, sToken, &amp;sState);
   15    ParseFree(pParser, free );
   16    TokenizerFree(pTokenizer);
   17    return sState.treeRoot;
@@ -197,10 +220,10 @@ simple.)
 We assume the existence of some kind of tokenizer which is created
 using TokenizerCreate() on line 8 and deleted by TokenizerFree()
 on line 16.  The GetNextToken() function on line 11 retrieves the
-next token from the input file and puts its type in the 
+next token from the input file and puts its type in the
 integer variable hTokenId.  The sToken variable is assumed to be
 some kind of structure that contains details about each token,
-such as its complete text, what line it occurs on, etc. </p>
+such as its complete text, what line it occurs on, etc.</p>

 <p>This example also assumes the existence of structure of type
 ParserState that holds state information about a particular parse.
@@ -217,7 +240,7 @@ tree.</p>
 <pre>
   ParseFile(){
      pParser = ParseAlloc( malloc );
-      while( GetNextToken(pTokenizer,&hTokenId, &sToken) ){
+      while( GetNextToken(pTokenizer,&amp;hTokenId, &amp;sToken) ){
         Parse(pParser, hTokenId, sToken);
      }
      Parse(pParser, 0, sToken);
@@ -277,25 +300,25 @@ specifies additional information Lemon requires to do its job.
 Most of the work in using Lemon is in writing an appropriate
 grammar file.</p>

-<p>The grammar file for lemon is, for the most part, free format.
+<p>The grammar file for Lemon is, for the most part, free format.
 It does not have sections or divisions like yacc or bison.  Any
 declaration can occur at any point in the file.
 Lemon ignores whitespace (except where it is needed to separate
-tokens) and it honors the same commenting conventions as C and C++.</p>
+tokens), and it honors the same commenting conventions as C and C++.</p>

 <h3>Terminals and Nonterminals</h3>

 <p>A terminal symbol (token) is any string of alphanumeric
 and/or underscore characters
-that begins with an upper case letter.
+that begins with an uppercase letter.
 A terminal can contain lowercase letters after the first character,
-but the usual convention is to make terminals all upper case.
+but the usual convention is to make terminals all uppercase.
 A nonterminal, on the other hand, is any string of alphanumeric
-and underscore characters than begins with a lower case letter.
-Again, the usual convention is to make nonterminals use all lower
-case letters.</p>
+and underscore characters than begins with a lowercase letter.
+Again, the usual convention is to make nonterminals use all lowercase
+letters.</p>

-<p>In Lemon, terminal and nonterminal symbols do not need to 
+<p>In Lemon, terminal and nonterminal symbols do not need to
 be declared or identified in a separate section of the grammar file.
 Lemon is able to generate a list of all terminals and nonterminals
 by examining the grammar rules, and it can always distinguish a
@@ -319,7 +342,8 @@ The list of terminals and nonterminals on the right-hand side of the
 rule can be empty.
 Rules can occur in any order, except that the left-hand side of the
 first rule is assumed to be the start symbol for the grammar (unless
-specified otherwise using the <tt>%start</tt> directive described below.)
+specified otherwise using the <tt><a href='#start_symbol'>%start_symbol</a></tt>
+directive described below.)
 A typical sequence of grammar rules might look something like this:
 <pre>
  expr ::= expr PLUS expr.
@@ -362,7 +386,7 @@ names to each symbol in a grammar rule and then using those symbolic
 names in the action.
 In yacc or bison, one would write this:
 <pre>
-  expr -> expr PLUS expr  { $$ = $1 + $3; };
+  expr -&gt; expr PLUS expr  { $$ = $1 + $3; };
 </pre>
 But in Lemon, the same rule becomes the following:
 <pre>
@@ -402,14 +426,14 @@ of the shift, and a reduce-reduce conflict is resolved by reducing
 whichever rule comes first in the grammar file.</p>

 <p>Just like in
-yacc and bison, Lemon allows a measure of control 
-over the resolution of paring conflicts using precedence rules.
+yacc and bison, Lemon allows a measure of control
+over the resolution of parsing conflicts using precedence rules.
 A precedence value can be assigned to any terminal symbol
-using the 
-<a href='#pleft'>%left</a>,
-<a href='#pright'>%right</a> or
-<a href='#pnonassoc'>%nonassoc</a> directives.  Terminal symbols
-mentioned in earlier directives have a lower precedence that
+using the
+<tt><a href='#pleft'>%left</a></tt>,
+<tt><a href='#pright'>%right</a></tt> or
+<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives.  Terminal symbols
+mentioned in earlier directives have a lower precedence than
 terminal symbols mentioned in later directives.  For example:</p>

 <p><pre>
@@ -485,29 +509,29 @@ as follows:
 <li> If the precedence of the token to be shifted is greater than
     the precedence of the rule to reduce, then resolve in favor
     of the shift.  No parsing conflict is reported.
-<li> If the precedence of the token it be shifted is less than the
+<li> If the precedence of the token to be shifted is less than the
     precedence of the rule to reduce, then resolve in favor of the
     reduce action.  No parsing conflict is reported.
 <li> If the precedences are the same and the shift token is
     right-associative, then resolve in favor of the shift.
     No parsing conflict is reported.
-<li> If the precedences are the same the shift token is
+<li> If the precedences are the same and the shift token is
     left-associative, then resolve in favor of the reduce.
     No parsing conflict is reported.
-<li> Otherwise, resolve the conflict by doing the shift and
-     report the parsing conflict.
+<li> Otherwise, resolve the conflict by doing the shift, and
+     report a parsing conflict.
 </ul>
 Reduce-reduce conflicts are resolved this way:
 <ul>
-<li> If either reduce rule 
+<li> If either reduce rule
     lacks precedence information, then resolve in favor of the
-     rule that appears first in the grammar and report a parsing
+     rule that appears first in the grammar, and report a parsing
     conflict.
-<li> If both rules have precedence and the precedence is different
+<li> If both rules have precedence and the precedence is different,
     then resolve the dispute in favor of the rule with the highest
-     precedence and do not report a conflict.
+     precedence, and do not report a conflict.
 <li> Otherwise, resolve the conflict by reducing by the rule that
-     appears first in the grammar and report a parsing conflict.
+     appears first in the grammar, and report a parsing conflict.
 </ul>

 <h3>Special Directives</h3>
@@ -516,40 +540,40 @@ Reduce-reduce conflicts are resolved this way:
 directives.  We've described all the grammar rules, so now we'll
 talk about the special directives.</p>

-<p>Directives in lemon can occur in any order.  You can put them before
-the grammar rules, or after the grammar rules, or in the mist of the
+<p>Directives in Lemon can occur in any order.  You can put them before
+the grammar rules, or after the grammar rules, or in the midst of the
 grammar rules.  It doesn't matter.  The relative order of
 directives used to assign precedence to terminals is important, but
 other than that, the order of directives in Lemon is arbitrary.</p>

 <p>Lemon supports the following special directives:
 <ul>
-<li><tt>%code</tt>
-<li><tt>%default_destructor</tt>
-<li><tt>%default_type</tt>
-<li><tt>%destructor</tt>
-<li><tt>%endif</tt>
-<li><tt>%extra_argument</tt>
-<li><tt>%fallback</tt>
-<li><tt>%ifdef</tt>
-<li><tt>%ifndef</tt>
-<li><tt>%include</tt>
-<li><tt>%left</tt>
-<li><tt>%name</tt>
-<li><tt>%nonassoc</tt>
-<li><tt>%parse_accept</tt>
-<li><tt>%parse_failure </tt>
-<li><tt>%right</tt>
-<li><tt>%stack_overflow</tt>
-<li><tt>%stack_size</tt>
-<li><tt>%start_symbol</tt>
-<li><tt>%syntax_error</tt>
-<li><tt>%token_class</tt>
-<li><tt>%token_destructor</tt>
-<li><tt>%token_prefix</tt>
-<li><tt>%token_type</tt>
-<li><tt>%type</tt>
-<li><tt>%wildcard</tt>
+<li><tt><a href='#pcode'>%code</a></tt>
+<li><tt><a href='#default_destructor'>%default_destructor</a></tt>
+<li><tt><a href='#default_type'>%default_type</a></tt>
+<li><tt><a href='#destructor'>%destructor</a></tt>
+<li><tt><a href='#pifdef'>%endif</a></tt>
+<li><tt><a href='#extraarg'>%extra_argument</a></tt>
+<li><tt><a href='#pfallback'>%fallback</a></tt>
+<li><tt><a href='#pifdef'>%ifdef</a></tt>
+<li><tt><a href='#pifdef'>%ifndef</a></tt>
+<li><tt><a href='#pinclude'>%include</a></tt>
+<li><tt><a href='#pleft'>%left</a></tt>
+<li><tt><a href='#pname'>%name</a></tt>
+<li><tt><a href='#pnonassoc'>%nonassoc</a></tt>
+<li><tt><a href='#parse_accept'>%parse_accept</a></tt>
+<li><tt><a href='#parse_failure'>%parse_failure</a></tt>
+<li><tt><a href='#pright'>%right</a></tt>
+<li><tt><a href='#stack_overflow'>%stack_overflow</a></tt>
+<li><tt><a href='#stack_size'>%stack_size</a></tt>
+<li><tt><a href='#start_symbol'>%start_symbol</a></tt>
+<li><tt><a href='#syntax_error'>%syntax_error</a></tt>
+<li><tt><a href='#token_class'>%token_class</a></tt>
+<li><tt><a href='#token_destructor'>%token_destructor</a></tt>
+<li><tt><a href='#token_prefix'>%token_prefix</a></tt>
+<li><tt><a href='#token_type'>%token_type</a></tt>
+<li><tt><a href='#ptype'>%type</a></tt>
+<li><tt><a href='#pwildcard'>%wildcard</a></tt>
 </ul>
 Each of these directives will be described separately in the
 following sections:</p>
@@ -557,43 +581,42 @@ following sections:</p>
 <a name='pcode'></a>
 <h4>The <tt>%code</tt> directive</h4>

-<p>The %code directive is used to specify addition C code that
+<p>The <tt>%code</tt> directive is used to specify additional C code that
 is added to the end of the main output file.  This is similar to
-the <a href='#pinclude'>%include</a> directive except that %include
-is inserted at the beginning of the main output file.</p>
+the <tt><a href='#pinclude'>%include</a></tt> directive except that
+<tt>%include</tt> is inserted at the beginning of the main output file.</p>

-<p>%code is typically used to include some action routines or perhaps
-a tokenizer or even the "main()" function 
+<p><tt>%code</tt> is typically used to include some action routines or perhaps
+a tokenizer or even the "main()" function
 as part of the output file.</p>

 <a name='default_destructor'></a>
 <h4>The <tt>%default_destructor</tt> directive</h4>

-<p>The %default_destructor directive specifies a destructor to 
+<p>The <tt>%default_destructor</tt> directive specifies a destructor to
 use for non-terminals that do not have their own destructor
-specified by a separate %destructor directive.  See the documentation
-on the <a name='#destructor'>%destructor</a> directive below for
+specified by a separate <tt>%destructor</tt> directive.  See the documentation
+on the <tt><a name='#destructor'>%destructor</a></tt> directive below for
 additional information.</p>

-<p>In some grammers, many different non-terminal symbols have the
-same datatype and hence the same destructor.  This directive is
-a convenience way to specify the same destructor for all those
+<p>In some grammars, many different non-terminal symbols have the
+same data type and hence the same destructor.  This directive is
+a convenient way to specify the same destructor for all those
 non-terminals using a single statement.</p>

 <a name='default_type'></a>
 <h4>The <tt>%default_type</tt> directive</h4>

-<p>The %default_type directive specifies the datatype of non-terminal
-symbols that do no have their own datatype defined using a separate
-<a href='#ptype'>%type</a> directive.  
-</p>
+<p>The <tt>%default_type</tt> directive specifies the data type of non-terminal
+symbols that do not have their own data type defined using a separate
+<tt><a href='#ptype'>%type</a></tt> directive.</p>

 <a name='destructor'></a>
 <h4>The <tt>%destructor</tt> directive</h4>

-<p>The %destructor directive is used to specify a destructor for
+<p>The <tt>%destructor</tt> directive is used to specify a destructor for
 a non-terminal symbol.
-(See also the <a href='#token_destructor'>%token_destructor</a>
+(See also the <tt><a href='#token_destructor'>%token_destructor</a></tt>
 directive which is used to specify a destructor for terminal symbols.)</p>

 <p>A non-terminal's destructor is called to dispose of the
@@ -615,7 +638,7 @@ or other resources held by that non-terminal.</p>
   %destructor nt { free($$); }
   nt(A) ::= ID NUM.   { A = malloc( 100 ); }
 </pre>
-This example is a bit contrived but it serves to illustrate how
+This example is a bit contrived, but it serves to illustrate how
 destructors work.  The example shows a non-terminal named
 "nt" that holds values of type "void*".  When the rule for
 an "nt" reduces, it sets the value of the non-terminal to
@@ -631,17 +654,17 @@ stack, unless the non-terminal is used in a C-code action.  If
 the non-terminal is used by C-code, then it is assumed that the
 C-code will take care of destroying it.
 More commonly, the value is used to build some
-larger structure and we don't want to destroy it, which is why
+larger structure, and we don't want to destroy it, which is why
 the destructor is not called in this circumstance.</p>

 <p>Destructors help avoid memory leaks by automatically freeing
 allocated objects when they go out of scope.
 To do the same using yacc or bison is much more difficult.</p>

-<a name="extraarg"></a>
+<a name='extraarg'></a>
 <h4>The <tt>%extra_argument</tt> directive</h4>

-The %extra_argument directive instructs Lemon to add a 4th parameter
+The <tt>%extra_argument</tt> directive instructs Lemon to add a 4th parameter
 to the parameter list of the Parse() function it generates.  Lemon
 doesn't do anything itself with this extra argument, but it does
 make the argument available to C-code action routines, destructors,
@@ -659,61 +682,64 @@ in the most recent call to Parse().</p>
 <a name='pfallback'></a>
 <h4>The <tt>%fallback</tt> directive</h4>

-<p>The %fallback directive specifies an alternative meaning for one
+<p>The <tt>%fallback</tt> directive specifies an alternative meaning for one
 or more tokens.  The alternative meaning is tried if the original token
-would have generated a syntax error.
+would have generated a syntax error.</p>

-<p>The %fallback directive was added to support robust parsing of SQL
-syntax in <a href="https://www.sqlite.org/">SQLite</a>.
+<p>The <tt>%fallback</tt> directive was added to support robust parsing of SQL
+syntax in <a href='https://www.sqlite.org/'>SQLite</a>.
 The SQL language contains a large assortment of keywords, each of which
 appears as a different token to the language parser.  SQL contains so
-many keywords, that it can be difficult for programmers to keep up with
+many keywords that it can be difficult for programmers to keep up with
 them all.  Programmers will, therefore, sometimes mistakenly use an
-obscure language keyword for an identifier.  The %fallback directive
+obscure language keyword for an identifier.  The <tt>%fallback</tt> directive
 provides a mechanism to tell the parser:  "If you are unable to parse
-this keyword, try treating it as an identifier instead."
+this keyword, try treating it as an identifier instead."</p>

-<p>The syntax of %fallback is as follows:
+<p>The syntax of <tt>%fallback</tt> is as follows:

 <blockquote>
-<tt>%fallback</tt>  <i>ID</i> <i>TOKEN...</i> <b>.</b>
-</blockquote>
+<tt>%fallback</tt> <i>ID</i> <i>TOKEN...</i> <b>.</b>
+</blockquote></p>

-<p>In words, the %fallback directive is followed by a list of token names
-terminated by a period.  The first token name is the fallback token - the
+<p>In words, the <tt>%fallback</tt> directive is followed by a list of token
+names terminated by a period.
+The first token name is the fallback token &mdash; the
 token to which all the other tokens fall back to.  The second and subsequent
 arguments are tokens which fall back to the token identified by the first
-argument.
+argument.</p>

 <a name='pifdef'></a>
-<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4>
+<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives</h4>

-<p>The %ifdef, %ifndef, and %endif directives are similar to
-#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general.
+<p>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives
+are similar to #ifdef, #ifndef, and #endif in the C-preprocessor,
+just not as general.
 Each of these directives must begin at the left margin.  No whitespace
-is allowed between the "%" and the directive name.
+is allowed between the "%" and the directive name.</p>

-<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is
+<p>Grammar text in between "<tt>%ifdef MACRO</tt>" and the next nested
+"<tt>%endif</tt>" is
 ignored unless the "-DMACRO" command-line option is used.  Grammar text
-betwen "%ifndef MACRO" and the next nested "%endif" is included except when
-the "-DMACRO" command-line option is used.
+betwen "<tt>%ifndef MACRO</tt>" and the next nested "<tt>%endif</tt>" is
+included except when the "-DMACRO" command-line option is used.</p>

-<p>Note that the argument to %ifdef and %ifndef must be a single 
-preprocessor symbol name, not a general expression.  There is no "%else"
-directive.
+<p>Note that the argument to <tt>%ifdef</tt> and <tt>%ifndef</tt> must
+be a single preprocessor symbol name, not a general expression.
+There is no "<tt>%else</tt>" directive.</p>


 <a name='pinclude'></a>
 <h4>The <tt>%include</tt> directive</h4>

-<p>The %include directive specifies C code that is included at the
-top of the generated parser.  You can include any text you want --
+<p>The <tt>%include</tt> directive specifies C code that is included at the
+top of the generated parser.  You can include any text you want &mdash;
 the Lemon parser generator copies it blindly.  If you have multiple
-%include directives in your grammar file, their values are concatenated
-so that all %include code ultimately appears near the top of the
-generated parser, in the same order as it appeared in the grammer.</p>
+<tt>%include</tt> directives in your grammar file, their values are concatenated
+so that all <tt>%include</tt> code ultimately appears near the top of the
+generated parser, in the same order as it appeared in the grammar.</p>

-<p>The %include directive is very handy for getting some extra #include
+<p>The <tt>%include</tt> directive is very handy for getting some extra #include
 preprocessor statements at the beginning of the generated parser.
 For example:</p>

@@ -722,17 +748,19 @@ For example:</p>
 </pre></p>

 <p>This might be needed, for example, if some of the C actions in the
-grammar call functions that are prototyed in unistd.h.</p>
+grammar call functions that are prototyped in unistd.h.</p>

 <a name='pleft'></a>
 <h4>The <tt>%left</tt> directive</h4>

-The %left directive is used (along with the <a href='#pright'>%right</a> and
-<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of 
-terminal symbols.  Every terminal symbol whose name appears after
-a %left directive but before the next period (".") is
+The <tt>%left</tt> directive is used (along with the
+<tt><a href='#pright'>%right</a></tt> and
+<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives) to declare
+precedences of terminal symbols.
+Every terminal symbol whose name appears after
+a <tt>%left</tt> directive but before the next period (".") is
 given the same left-associative precedence value.  Subsequent
-%left directives have higher precedence.  For example:</p>
+<tt>%left</tt> directives have higher precedence.  For example:</p>

 <p><pre>
   %left AND.
@@ -743,20 +771,21 @@ given the same left-associative precedence value.  Subsequent
   %right EXP NOT.
 </pre></p>

-<p>Note the period that terminates each %left, %right or %nonassoc
+<p>Note the period that terminates each <tt>%left</tt>,
+<tt>%right</tt> or <tt>%nonassoc</tt>
 directive.</p>

 <p>LALR(1) grammars can get into a situation where they require
 a large amount of stack space if you make heavy use or right-associative
-operators.  For this reason, it is recommended that you use %left
-rather than %right whenever possible.</p>
+operators.  For this reason, it is recommended that you use <tt>%left</tt>
+rather than <tt>%right</tt> whenever possible.</p>

 <a name='pname'></a>
 <h4>The <tt>%name</tt> directive</h4>

 <p>By default, the functions generated by Lemon all begin with the
 five-character string "Parse".  You can change this string to something
-different using the %name directive.  For instance:</p>
+different using the <tt>%name</tt> directive.  For instance:</p>

 <p><pre>
   %name Abcde
@@ -770,22 +799,22 @@ functions named
 <li> AbcdeTrace(), and
 <li> Abcde().
 </ul>
-The %name directive allows you to generator two or more different
-parsers and link them all into the same executable.
-</p>
+The <tt>%name</tt> directive allows you to generate two or more different
+parsers and link them all into the same executable.</p>

 <a name='pnonassoc'></a>
 <h4>The <tt>%nonassoc</tt> directive</h4>

 <p>This directive is used to assign non-associative precedence to
-one or more terminal symbols.  See the section on 
+one or more terminal symbols.  See the section on
 <a href='#precrules'>precedence rules</a>
-or on the <a href='#pleft'>%left</a> directive for additional information.</p>
+or on the <tt><a href='#pleft'>%left</a></tt> directive
+for additional information.</p>

 <a name='parse_accept'></a>
 <h4>The <tt>%parse_accept</tt> directive</h4>

-<p>The %parse_accept directive specifies a block of C code that is
+<p>The <tt>%parse_accept</tt> directive specifies a block of C code that is
 executed whenever the parser accepts its input string.  To "accept"
 an input string means that the parser was able to process all tokens
 without error.</p>
@@ -801,7 +830,7 @@ without error.</p>
 <a name='parse_failure'></a>
 <h4>The <tt>%parse_failure</tt> directive</h4>

-<p>The %parse_failure directive specifies a block of C code that
+<p>The <tt>%parse_failure</tt> directive specifies a block of C code that
 is executed whenever the parser fails complete.  This code is not
 executed until the parser has tried and failed to resolve an input
 error using is usual error recovery strategy.  The routine is
@@ -817,14 +846,14 @@ only invoked when parsing is unable to continue.</p>
 <h4>The <tt>%right</tt> directive</h4>

 <p>This directive is used to assign right-associative precedence to
-one or more terminal symbols.  See the section on 
+one or more terminal symbols.  See the section on
 <a href='#precrules'>precedence rules</a>
 or on the <a href='#pleft'>%left</a> directive for additional information.</p>

 <a name='stack_overflow'></a>
 <h4>The <tt>%stack_overflow</tt> directive</h4>

-<p>The %stack_overflow directive specifies a block of C code that
+<p>The <tt>%stack_overflow</tt> directive specifies a block of C code that
 is executed if the parser's internal stack ever overflows.  Typically
 this just prints an error message.  After a stack overflow, the parser
 will be unable to continue and must be reset.</p>
@@ -837,7 +866,7 @@ will be unable to continue and must be reset.</p>

 <p>You can help prevent parser stack overflows by avoiding the use
 of right recursion and right-precedence operators in your grammar.
-Use left recursion and and left-precedence operators instead, to
+Use left recursion and and left-precedence operators instead to
 encourage rules to reduce sooner and keep the stack size down.
 For example, do rules like this:
 <pre>
@@ -848,7 +877,7 @@ Not like this:
 <pre>
   list ::= element list.      // right-recursion.  Bad!
   list ::= .
-</pre>
+</pre></p>

 <a name='stack_size'></a>
 <h4>The <tt>%stack_size</tt> directive</h4>
@@ -856,7 +885,7 @@ Not like this:
 <p>If stack overflow is a problem and you can't resolve the trouble
 by using left-recursion, then you might want to increase the size
 of the parser's stack using this directive.  Put an positive integer
-after the %stack_size directive and Lemon will generate a parse
+after the <tt>%stack_size</tt> directive and Lemon will generate a parse
 with a stack of the requested size.  The default value is 100.</p>

 <p><pre>
@@ -866,25 +895,40 @@ with a stack of the requested size.  The default value is 100.</p>
 <a name='start_symbol'></a>
 <h4>The <tt>%start_symbol</tt> directive</h4>

-<p>By default, the start-symbol for the grammar that Lemon generates
+<p>By default, the start symbol for the grammar that Lemon generates
 is the first non-terminal that appears in the grammar file.  But you
-can choose a different start-symbol using the %start_symbol directive.</p>
+can choose a different start symbol using the
+<tt>%start_symbol</tt> directive.</p>

 <p><pre>
   %start_symbol  prog
 </pre></p>

+<a name='syntax_error'></a>
+<h4>The <tt>%syntax_error</tt> directive</h4>
+
+<p>See <a href='#error_processing'>Error Processing</a>.</p>
+
+<a name='token_class'></a>
+<h4>The <tt>%token_class</tt> directive</h4>
+
+<p>Undocumented.  Appears to be related to the MULTITERMINAL concept.
+<a href='http://sqlite.org/src/fdiff?v1=796930d5fc2036c7&v2=624b24c5dc048e09&sbs=0'>Implementation</a>.</p>
+
 <a name='token_destructor'></a>
 <h4>The <tt>%token_destructor</tt> directive</h4>

-<p>The %destructor directive assigns a destructor to a non-terminal
-symbol.  (See the description of the %destructor directive above.)
-This directive does the same thing for all terminal symbols.</p>
+<p>The <tt>%destructor</tt> directive assigns a destructor to a non-terminal
+symbol.  (See the description of the
+<tt><a href='%destructor'>%destructor</a></tt> directive above.)
+The <tt>%token_destructor</tt> directive does the same thing
+for all terminal symbols.</p>

 <p>Unlike non-terminal symbols which may each have a different data type
 for their values, terminals all use the same data type (defined by
-the %token_type directive) and so they use a common destructor.  Other
-than that, the token destructor works just like the non-terminal
+the <tt><a href='#token_type'>%token_type</a></tt> directive)
+and so they use a common destructor.
+Other than that, the token destructor works just like the non-terminal
 destructors.</p>

 <a name='token_prefix'></a>
@@ -893,8 +937,9 @@ destructors.</p>
 <p>Lemon generates #defines that assign small integer constants
 to each terminal symbol in the grammar.  If desired, Lemon will
 add a prefix specified by this directive
-to each of the #defines it generates.
-So if the default output of Lemon looked like this:
+to each of the #defines it generates.</p>
+
+<p>So if the default output of Lemon looked like this:
 <pre>
    #define AND              1
    #define MINUS            2
@@ -911,7 +956,7 @@ to cause Lemon to produce these symbols instead:
    #define TOKEN_MINUS      2
    #define TOKEN_OR         3
    #define TOKEN_PLUS       4
-</pre>
+</pre></p>

 <a name='token_type'></a><a name='ptype'></a>
 <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
@@ -929,10 +974,10 @@ token structure.  Like this:</p>
 </pre></p>

 <p>If the data type of terminals is not specified, the default value
-is "int".</p>
+is "void*".</p>

 <p>Non-terminal symbols can each have their own data types.  Typically
-the data type  of a non-terminal is a pointer to the root of a parse-tree
+the data type of a non-terminal is a pointer to the root of a parse tree
 structure that contains all information about that non-terminal.
 For example:</p>

@@ -953,14 +998,15 @@ and able to pay that price, fine.  You just need to know.</p>
 <a name='pwildcard'></a>
 <h4>The <tt>%wildcard</tt> directive</h4>

-<p>The %wildcard directive is followed by a single token name and a
-period.  This directive specifies that the identified token should 
-match any input token.
+<p>The <tt>%wildcard</tt> directive is followed by a single token name and a
+period.  This directive specifies that the identified token should
+match any input token.</p>

 <p>When the generated parser has the choice of matching an input against
 the wildcard token and some other token, the other token is always used.
-The wildcard token is only matched if there are no other alternatives.
+The wildcard token is only matched if there are no alternatives.</p>

+<a name='error_processing'></a>
 <h3>Error Processing</h3>

 <p>After extensive experimentation over several years, it has been
@@ -968,19 +1014,20 @@ discovered that the error recovery strategy used by yacc is about
 as good as it gets.  And so that is what Lemon uses.</p>

 <p>When a Lemon-generated parser encounters a syntax error, it
-first invokes the code specified by the %syntax_error directive, if
+first invokes the code specified by the <tt>%syntax_error</tt> directive, if
 any.  It then enters its error recovery strategy.  The error recovery
 strategy is to begin popping the parsers stack until it enters a
 state where it is permitted to shift a special non-terminal symbol
 named "error".  It then shifts this non-terminal and continues
-parsing.  But the %syntax_error routine will not be called again
+parsing.  The <tt>%syntax_error</tt> routine will not be called again
 until at least three new tokens have been successfully shifted.</p>

 <p>If the parser pops its stack until the stack is empty, and it still
-is unable to shift the error symbol, then the %parse_failed routine
+is unable to shift the error symbol, then the
+<tt><a href='#parse_failure'>%parse_failure</a></tt> routine
 is invoked and the parser resets itself to its start state, ready
 to begin parsing a new file.  This is what will happen at the very
-first syntax error, of course, if there are no instances of the 
+first syntax error, of course, if there are no instances of the
 "error" non-terminal in your grammar.</p>

 </body>
@@ -0,0 +1,8 @@
+## Loadable Extensions
+
+Various [loadable extensions](https://www.sqlite.org/loadext.html) for
+SQLite are found in subfolders.
+
+Most subfolders are dedicated to a single loadable extension (for
+example FTS5, or RTREE).  But the misc/ subfolder contains a collection
+of smaller single-file extensions.
@@ -1,2 +0,0 @@
-Version loadable extensions to SQLite are found in subfolders
-of this folder.
@@ -99,7 +99,11 @@ static void scalarFunc(

 #ifdef SQLITE_TEST

-#include <tcl.h>
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include <string.h>

 /*
@@ -349,8 +349,9 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
 ** Return the number of bytes read, or 0 on error.
 ** The value is stored in *v.
 */
-int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
-  const char *pStart = p;
+int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
+  const unsigned char *p = (const unsigned char*)pBuf;
+  const unsigned char *pStart = p;
  u32 a;
  u64 b;
  int shift;
@@ -371,8 +372,8 @@ int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
 }

 /*
-** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
-** 32-bit integer before it is returned.
+** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to 
+** a non-negative 32-bit integer before it is returned.
 */
 int sqlite3Fts3GetVarint32(const char *p, int *pi){
  u32 a;
@@ -388,7 +389,9 @@ int sqlite3Fts3GetVarint32(const char *p, int *pi){
  GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *pi, 3);
  GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4);
  a = (a & 0x0FFFFFFF );
-  *pi = (int)(a | ((u32)(*p & 0x0F) << 28));
+  *pi = (int)(a | ((u32)(*p & 0x07) << 28));
+  assert( 0==(a & 0x80000000) );
+  assert( *pi>=0 );
  return 5;
 }

@@ -492,6 +495,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
  assert( p->pSegments==0 );

  /* Free any prepared statements held */
+  sqlite3_finalize(p->pSeekStmt);
  for(i=0; i<SizeofArray(p->aStmt); i++){
    sqlite3_finalize(p->aStmt[i]);
  }
@@ -1217,65 +1221,66 @@ static int fts3InitVtab(
            break;
          }
        }
-        if( iOpt==SizeofArray(aFts4Opt) ){
-          sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
-          rc = SQLITE_ERROR;
-        }else{
-          switch( iOpt ){
-            case 0:               /* MATCHINFO */
-              if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
-                sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
-                rc = SQLITE_ERROR;
-              }
-              bNoDocsize = 1;
-              break;
+        switch( iOpt ){
+          case 0:               /* MATCHINFO */
+            if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
+              sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
+              rc = SQLITE_ERROR;
+            }
+            bNoDocsize = 1;
+            break;

-            case 1:               /* PREFIX */
-              sqlite3_free(zPrefix);
-              zPrefix = zVal;
-              zVal = 0;
-              break;
+          case 1:               /* PREFIX */
+            sqlite3_free(zPrefix);
+            zPrefix = zVal;
+            zVal = 0;
+            break;

-            case 2:               /* COMPRESS */
-              sqlite3_free(zCompress);
-              zCompress = zVal;
-              zVal = 0;
-              break;
+          case 2:               /* COMPRESS */
+            sqlite3_free(zCompress);
+            zCompress = zVal;
+            zVal = 0;
+            break;

-            case 3:               /* UNCOMPRESS */
-              sqlite3_free(zUncompress);
-              zUncompress = zVal;
-              zVal = 0;
-              break;
+          case 3:               /* UNCOMPRESS */
+            sqlite3_free(zUncompress);
+            zUncompress = zVal;
+            zVal = 0;
+            break;

-            case 4:               /* ORDER */
-              if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
-               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
-              ){
-                sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
-                rc = SQLITE_ERROR;
-              }
-              bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
-              break;
+          case 4:               /* ORDER */
+            if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
+             && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
+            ){
+              sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
+              rc = SQLITE_ERROR;
+            }
+            bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
+            break;

-            case 5:              /* CONTENT */
-              sqlite3_free(zContent);
-              zContent = zVal;
-              zVal = 0;
-              break;
+          case 5:              /* CONTENT */
+            sqlite3_free(zContent);
+            zContent = zVal;
+            zVal = 0;
+            break;

-            case 6:              /* LANGUAGEID */
-              assert( iOpt==6 );
-              sqlite3_free(zLanguageid);
-              zLanguageid = zVal;
-              zVal = 0;
-              break;
+          case 6:              /* LANGUAGEID */
+            assert( iOpt==6 );
+            sqlite3_free(zLanguageid);
+            zLanguageid = zVal;
+            zVal = 0;
+            break;

-            case 7:              /* NOTINDEXED */
-              azNotindexed[nNotindexed++] = zVal;
-              zVal = 0;
-              break;
-          }
+          case 7:              /* NOTINDEXED */
+            azNotindexed[nNotindexed++] = zVal;
+            zVal = 0;
+            break;
+
+          default:
+            assert( iOpt==SizeofArray(aFts4Opt) );
+            sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
+            rc = SQLITE_ERROR;
+            break;
        }
        sqlite3_free(zVal);
      }
@@ -1363,9 +1368,9 @@ static int fts3InitVtab(
  p->pTokenizer = pTokenizer;
  p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  p->bHasDocsize = (isFts4 && bNoDocsize==0);
-  p->bHasStat = isFts4;
-  p->bFts4 = isFts4;
-  p->bDescIdx = bDescIdx;
+  p->bHasStat = (u8)isFts4;
+  p->bFts4 = (u8)isFts4;
+  p->bDescIdx = (u8)bDescIdx;
  p->nAutoincrmerge = 0xff;   /* 0xff means setting unknown */
  p->zContentTbl = zContent;
  p->zLanguageid = zLanguageid;
@@ -1396,7 +1401,9 @@ static int fts3InitVtab(
    char *z; 
    int n = 0;
    z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n);
-    memcpy(zCsr, z, n);
+    if( n>0 ){
+      memcpy(zCsr, z, n);
+    }
    zCsr[n] = '\0';
    sqlite3Fts3Dequote(zCsr);
    p->azColumn[iCol] = zCsr;
@@ -1680,6 +1687,39 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
  return SQLITE_OK;
 }

+/*
+** Finalize the statement handle at pCsr->pStmt.
+**
+** Or, if that statement handle is one created by fts3CursorSeekStmt(),
+** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement
+** pointer there instead of finalizing it.
+*/
+static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){
+  if( pCsr->bSeekStmt ){
+    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+    if( p->pSeekStmt==0 ){
+      p->pSeekStmt = pCsr->pStmt;
+      sqlite3_reset(pCsr->pStmt);
+      pCsr->pStmt = 0;
+    }
+    pCsr->bSeekStmt = 0;
+  }
+  sqlite3_finalize(pCsr->pStmt);
+}
+
+/*
+** Free all resources currently held by the cursor passed as the only
+** argument.
+*/
+static void fts3ClearCursor(Fts3Cursor *pCsr){
+  fts3CursorFinalizeStmt(pCsr);
+  sqlite3Fts3FreeDeferredTokens(pCsr);
+  sqlite3_free(pCsr->aDoclist);
+  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
+  sqlite3Fts3ExprFree(pCsr->pExpr);
+  memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
+}
+
 /*
 ** Close the cursor.  For additional information see the documentation
 ** on the xClose method of the virtual table interface.
@@ -1687,11 +1727,7 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
 static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
  Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
-  sqlite3_finalize(pCsr->pStmt);
-  sqlite3Fts3ExprFree(pCsr->pExpr);
-  sqlite3Fts3FreeDeferredTokens(pCsr);
-  sqlite3_free(pCsr->aDoclist);
-  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
+  fts3ClearCursor(pCsr);
  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  sqlite3_free(pCsr);
  return SQLITE_OK;
@@ -1705,20 +1741,23 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
 **
 ** (or the equivalent for a content=xxx table) and set pCsr->pStmt to
 ** it. If an error occurs, return an SQLite error code.
-**
-** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK.
 */
-static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
+static int fts3CursorSeekStmt(Fts3Cursor *pCsr){
  int rc = SQLITE_OK;
  if( pCsr->pStmt==0 ){
    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
    char *zSql;
-    zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
-    if( !zSql ) return SQLITE_NOMEM;
-    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
-    sqlite3_free(zSql);
+    if( p->pSeekStmt ){
+      pCsr->pStmt = p->pSeekStmt;
+      p->pSeekStmt = 0;
+    }else{
+      zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
+      if( !zSql ) return SQLITE_NOMEM;
+      rc = sqlite3_prepare_v3(p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
+      sqlite3_free(zSql);
+    }
+    if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1;
  }
-  *ppStmt = pCsr->pStmt;
  return rc;
 }

@@ -1730,9 +1769,7 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
 static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
  int rc = SQLITE_OK;
  if( pCsr->isRequireSeek ){
-    sqlite3_stmt *pStmt = 0;
-
-    rc = fts3CursorSeekStmt(pCsr, &pStmt);
+    rc = fts3CursorSeekStmt(pCsr);
    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
      pCsr->isRequireSeek = 0;
@@ -1821,7 +1858,8 @@ static int fts3ScanInteriorNode(
    isFirstTerm = 0;
    zCsr += fts3GetVarint32(zCsr, &nSuffix);
    
-    if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
+    assert( nPrefix>=0 && nSuffix>=0 );
+    if( &zCsr[nSuffix]>zEnd ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
@@ -2631,7 +2669,7 @@ int sqlite3Fts3FirstFilter(
    fts3ColumnlistCopy(0, &p);
  }

-  while( p<pEnd && *p==0x01 ){
+  while( p<pEnd ){
    sqlite3_int64 iCol;
    p++;
    p += sqlite3Fts3GetVarint(p, &iCol);
@@ -3190,11 +3228,7 @@ static int fts3FilterMethod(
  assert( iIdx==nVal );

  /* In case the cursor has been used before, clear it now. */
-  sqlite3_finalize(pCsr->pStmt);
-  sqlite3_free(pCsr->aDoclist);
-  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
-  sqlite3Fts3ExprFree(pCsr->pExpr);
-  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
+  fts3ClearCursor(pCsr);

  /* Set the lower and upper bounds on docids to return */
  pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
@@ -3252,13 +3286,13 @@ static int fts3FilterMethod(
      );
    }
    if( zSql ){
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+      rc = sqlite3_prepare_v3(p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
      sqlite3_free(zSql);
    }else{
      rc = SQLITE_NOMEM;
    }
  }else if( eSearch==FTS3_DOCID_SEARCH ){
-    rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt);
+    rc = fts3CursorSeekStmt(pCsr);
    if( rc==SQLITE_OK ){
      rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons);
    }
@@ -3273,7 +3307,12 @@ static int fts3FilterMethod(
 ** routine to find out if it has reached the end of a result set.
 */
 static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
-  return ((Fts3Cursor *)pCursor)->isEof;
+  Fts3Cursor *pCsr = (Fts3Cursor*)pCursor;
+  if( pCsr->isEof ){
+    fts3ClearCursor(pCsr);
+    pCsr->isEof = 1;
+  }
+  return pCsr->isEof;
 }

 /* 
@@ -3311,33 +3350,37 @@ static int fts3ColumnMethod(
  /* The column value supplied by SQLite must be in range. */
  assert( iCol>=0 && iCol<=p->nColumn+2 );

-  if( iCol==p->nColumn+1 ){
-    /* This call is a request for the "docid" column. Since "docid" is an 
-    ** alias for "rowid", use the xRowid() method to obtain the value.
-    */
-    sqlite3_result_int64(pCtx, pCsr->iPrevId);
-  }else if( iCol==p->nColumn ){
-    /* The extra column whose name is the same as the table.
-    ** Return a blob which is a pointer to the cursor.  */
-    sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
-  }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
-    sqlite3_result_int64(pCtx, pCsr->iLangid);
-  }else{
-    /* The requested column is either a user column (one that contains 
-    ** indexed data), or the language-id column.  */
-    rc = fts3CursorSeek(0, pCsr);
+  switch( iCol-p->nColumn ){
+    case 0:
+      /* The special 'table-name' column */
+      sqlite3_result_pointer(pCtx, pCsr, "fts3cursor", 0);
+      break;

-    if( rc==SQLITE_OK ){
-      if( iCol==p->nColumn+2 ){
-        int iLangid = 0;
-        if( p->zLanguageid ){
-          iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1);
-        }
-        sqlite3_result_int(pCtx, iLangid);
-      }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
+    case 1:
+      /* The docid column */
+      sqlite3_result_int64(pCtx, pCsr->iPrevId);
+      break;
+
+    case 2:
+      if( pCsr->pExpr ){
+        sqlite3_result_int64(pCtx, pCsr->iLangid);
+        break;
+      }else if( p->zLanguageid==0 ){
+        sqlite3_result_int(pCtx, 0);
+        break;
+      }else{
+        iCol = p->nColumn;
+        /* fall-through */
+      }
+
+    default:
+      /* A user column. Or, if this is a full-table scan, possibly the
+      ** language-id column. Seek the cursor. */
+      rc = fts3CursorSeek(0, pCsr);
+      if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){
        sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
      }
-    }
+      break;
  }

  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
@@ -3386,8 +3429,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
  const u32 nMinMerge = 64;       /* Minimum amount of incr-merge work to do */

  Fts3Table *p = (Fts3Table*)pVtab;
-  int rc = sqlite3Fts3PendingTermsFlush(p);
+  int rc;
+  i64 iLastRowid = sqlite3_last_insert_rowid(p->db);

+  rc = sqlite3Fts3PendingTermsFlush(p);
  if( rc==SQLITE_OK 
   && p->nLeafAdd>(nMinMerge/16) 
   && p->nAutoincrmerge && p->nAutoincrmerge!=0xff
@@ -3402,6 +3447,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
    if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge);
  }
  sqlite3Fts3SegmentsClose(p);
+  sqlite3_set_last_insert_rowid(p->db, iLastRowid);
  return rc;
 }

@@ -3414,17 +3460,11 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
 static int fts3SetHasStat(Fts3Table *p){
  int rc = SQLITE_OK;
  if( p->bHasStat==2 ){
-    const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'";
-    char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName);
-    if( zSql ){
-      sqlite3_stmt *pStmt = 0;
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-      if( rc==SQLITE_OK ){
-        int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW);
-        rc = sqlite3_finalize(pStmt);
-        if( rc==SQLITE_OK ) p->bHasStat = bHasStat;
-      }
-      sqlite3_free(zSql);
+    char *zTbl = sqlite3_mprintf("%s_stat", p->zName);
+    if( zTbl ){
+      int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0);
+      sqlite3_free(zTbl);
+      p->bHasStat = (res==SQLITE_OK);
    }else{
      rc = SQLITE_NOMEM;
    }
@@ -3531,18 +3571,17 @@ static int fts3FunctionArg(
  sqlite3_value *pVal,            /* argv[0] passed to function */
  Fts3Cursor **ppCsr              /* OUT: Store cursor handle here */
 ){
-  Fts3Cursor *pRet;
-  if( sqlite3_value_type(pVal)!=SQLITE_BLOB 
-   || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
-  ){
+  int rc;
+  *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal, "fts3cursor");
+  if( (*ppCsr)!=0 ){
+    rc = SQLITE_OK;
+  }else{
    char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
    sqlite3_result_error(pContext, zErr, -1);
    sqlite3_free(zErr);
-    return SQLITE_ERROR;
+    rc = SQLITE_ERROR;
  }
-  memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
-  *ppCsr = pRet;
-  return SQLITE_OK;
+  return rc;
 }

 /*
@@ -3929,7 +3968,7 @@ int sqlite3Fts3Init(sqlite3 *db){
 #endif

  /* Create the virtual table wrapper around the hash-table and overload 
-  ** the two scalar functions. If this is successful, register the
+  ** the four scalar functions. If this is successful, register the
  ** module with sqlite.
  */
  if( SQLITE_OK==rc 
@@ -4512,7 +4551,7 @@ static int fts3EvalIncrPhraseNext(
  ** one incremental token. In which case the bIncr flag is set. */
  assert( p->bIncr==1 );

-  if( p->nToken==1 && p->bIncr ){
+  if( p->nToken==1 ){
    rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, 
        &pDL->iDocid, &pDL->pList, &pDL->nList
    );
@@ -4745,6 +4784,7 @@ static void fts3EvalTokenCosts(
 ** the number of overflow pages consumed by a record B bytes in size.
 */
 static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
+  int rc = SQLITE_OK;
  if( pCsr->nRowAvg==0 ){
    /* The average document size, which is required to calculate the cost
    ** of each doclist, has not yet been determined. Read the required 
@@ -4757,7 +4797,6 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
    ** data stored in all rows of each column of the table, from left
    ** to right.
    */
-    int rc;
    Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
    sqlite3_stmt *pStmt;
    sqlite3_int64 nDoc = 0;
@@ -4784,11 +4823,10 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
    pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
    assert( pCsr->nRowAvg>0 ); 
    rc = sqlite3_reset(pStmt);
-    if( rc!=SQLITE_OK ) return rc;
  }

  *pnPage = pCsr->nRowAvg;
-  return SQLITE_OK;
+  return rc;
 }

 /*
@@ -5138,7 +5176,8 @@ static void fts3EvalNextRow(
          pExpr->iDocid = pLeft->iDocid;
          pExpr->bEof = (pLeft->bEof || pRight->bEof);
          if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){
-            if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){
+            assert( pRight->eType==FTSQUERY_PHRASE );
+            if( pRight->pPhrase->doclist.aAll ){
              Fts3Doclist *pDl = &pRight->pPhrase->doclist;
              while( *pRc==SQLITE_OK && pRight->bEof==0 ){
                memset(pDl->pList, 0, pDl->nList);
@@ -5167,7 +5206,7 @@ static void fts3EvalNextRow(

        if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
          fts3EvalNextRow(pCsr, pLeft, pRc);
-        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
+        }else if( pLeft->bEof || iCmp>0 ){
          fts3EvalNextRow(pCsr, pRight, pRc);
        }else{
          fts3EvalNextRow(pCsr, pLeft, pRc);
@@ -5259,7 +5298,6 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
  */
  if( *pRc==SQLITE_OK 
   && pExpr->eType==FTSQUERY_NEAR 
-   && pExpr->bEof==0
   && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
  ){
    Fts3Expr *p; 
@@ -5268,42 +5306,39 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){

    /* Allocate temporary working space. */
    for(p=pExpr; p->pLeft; p=p->pLeft){
+      assert( p->pRight->pPhrase->doclist.nList>0 );
      nTmp += p->pRight->pPhrase->doclist.nList;
    }
    nTmp += p->pPhrase->doclist.nList;
-    if( nTmp==0 ){
+    aTmp = sqlite3_malloc(nTmp*2);
+    if( !aTmp ){
+      *pRc = SQLITE_NOMEM;
      res = 0;
    }else{
-      aTmp = sqlite3_malloc(nTmp*2);
-      if( !aTmp ){
-        *pRc = SQLITE_NOMEM;
-        res = 0;
-      }else{
-        char *aPoslist = p->pPhrase->doclist.pList;
-        int nToken = p->pPhrase->nToken;
+      char *aPoslist = p->pPhrase->doclist.pList;
+      int nToken = p->pPhrase->nToken;

-        for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
-          Fts3Phrase *pPhrase = p->pRight->pPhrase;
-          int nNear = p->nNear;
-          res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
-        }
-
-        aPoslist = pExpr->pRight->pPhrase->doclist.pList;
-        nToken = pExpr->pRight->pPhrase->nToken;
-        for(p=pExpr->pLeft; p && res; p=p->pLeft){
-          int nNear;
-          Fts3Phrase *pPhrase;
-          assert( p->pParent && p->pParent->pLeft==p );
-          nNear = p->pParent->nNear;
-          pPhrase = (
-              p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
-              );
-          res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
-        }
+      for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
+        Fts3Phrase *pPhrase = p->pRight->pPhrase;
+        int nNear = p->nNear;
+        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
      }

-      sqlite3_free(aTmp);
+      aPoslist = pExpr->pRight->pPhrase->doclist.pList;
+      nToken = pExpr->pRight->pPhrase->nToken;
+      for(p=pExpr->pLeft; p && res; p=p->pLeft){
+        int nNear;
+        Fts3Phrase *pPhrase;
+        assert( p->pParent && p->pParent->pLeft==p );
+        nNear = p->pParent->nNear;
+        pPhrase = (
+            p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
+        );
+        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
+      }
    }
+
+    sqlite3_free(aTmp);
  }

  return res;
@@ -230,6 +230,7 @@ struct Fts3Table {
  ** statements is run and reset within a single virtual table API call. 
  */
  sqlite3_stmt *aStmt[40];
+  sqlite3_stmt *pSeekStmt;        /* Cache for fts3CursorSeekStmt() */

  char *zReadExprlist;
  char *zWriteExprlist;
@@ -299,6 +300,7 @@ struct Fts3Cursor {
  i16 eSearch;                    /* Search strategy (see below) */
  u8 isEof;                       /* True if at End Of Results */
  u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
+  u8 bSeekStmt;                   /* True if pStmt is a seek */
  sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
  Fts3Expr *pExpr;                /* Parsed MATCH query string */
  int iLangid;                    /* Language being queried for */
@@ -18,7 +18,14 @@
 ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
 */

-#include <tcl.h>
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 #include <string.h>
 #include <assert.h>

@@ -143,7 +150,7 @@ static int nm_match_count(
 /*
 ** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
 */
-static int fts3_near_match_cmd(
+static int SQLITE_TCLAPI fts3_near_match_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
@@ -278,7 +285,7 @@ static int fts3_near_match_cmd(
 **    # Restore initial incr-load settings:
 **    eval fts3_configure_incr_load $cfg
 */
-static int fts3_configure_incr_load_cmd(
+static int SQLITE_TCLAPI fts3_configure_incr_load_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
@@ -488,7 +495,7 @@ static int testTokenizerLanguage(
 }
 #endif

-static int fts3_test_tokenizer_cmd(
+static int SQLITE_TCLAPI fts3_test_tokenizer_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
@@ -517,7 +524,7 @@ static int fts3_test_tokenizer_cmd(
  return TCL_OK;
 }

-static int fts3_test_varint_cmd(
+static int SQLITE_TCLAPI fts3_test_varint_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
@@ -224,7 +224,11 @@ int sqlite3Fts3InitTokenizer(

 #ifdef SQLITE_TEST

-#include <tcl.h>
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include <string.h>

 /*
@@ -136,16 +136,16 @@ static int unicodeAddExceptions(
 ){
  const unsigned char *z = (const unsigned char *)zIn;
  const unsigned char *zTerm = &z[nIn];
-  int iCode;
+  unsigned int iCode;
  int nEntry = 0;

  assert( bAlnum==0 || bAlnum==1 );

  while( z<zTerm ){
    READ_UTF8(z, zTerm, iCode);
-    assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
-    if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 
-     && sqlite3FtsUnicodeIsdiacritic(iCode)==0 
+    assert( (sqlite3FtsUnicodeIsalnum((int)iCode) & 0xFFFFFFFE)==0 );
+    if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 
+     && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0 
    ){
      nEntry++;
    }
@@ -162,13 +162,13 @@ static int unicodeAddExceptions(
    z = (const unsigned char *)zIn;
    while( z<zTerm ){
      READ_UTF8(z, zTerm, iCode);
-      if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 
-       && sqlite3FtsUnicodeIsdiacritic(iCode)==0
+      if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 
+       && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
      ){
        int i, j;
-        for(i=0; i<nNew && aNew[i]<iCode; i++);
+        for(i=0; i<nNew && aNew[i]<(int)iCode; i++);
        for(j=nNew; j>i; j--) aNew[j] = aNew[j-1];
-        aNew[i] = iCode;
+        aNew[i] = (int)iCode;
        nNew++;
      }
    }
@@ -318,7 +318,7 @@ static int unicodeNext(
 ){
  unicode_cursor *pCsr = (unicode_cursor *)pC;
  unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer);
-  int iCode = 0;
+  unsigned int iCode = 0;
  char *zOut;
  const unsigned char *z = &pCsr->aInput[pCsr->iOff];
  const unsigned char *zStart = z;
@@ -330,7 +330,7 @@ static int unicodeNext(
  ** the input.  */
  while( z<zTerm ){
    READ_UTF8(z, zTerm, iCode);
-    if( unicodeIsAlnum(p, iCode) ) break;
+    if( unicodeIsAlnum(p, (int)iCode) ) break;
    zStart = z;
  }
  if( zStart>=zTerm ) return SQLITE_DONE;
@@ -350,7 +350,7 @@ static int unicodeNext(

    /* Write the folded case of the last character read to the output */
    zEnd = z;
-    iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic);
+    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
    if( iOut ){
      WRITE_UTF8(zOut, iOut);
    }
@@ -358,8 +358,8 @@ static int unicodeNext(
    /* If the cursor is not at EOF, read the next character */
    if( z>=zTerm ) break;
    READ_UTF8(z, zTerm, iCode);
-  }while( unicodeIsAlnum(p, iCode) 
-       || sqlite3FtsUnicodeIsdiacritic(iCode)
+  }while( unicodeIsAlnum(p, (int)iCode) 
+       || sqlite3FtsUnicodeIsdiacritic((int)iCode)
  );

  /* Set the output variables and return. */
@@ -127,9 +127,9 @@ int sqlite3FtsUnicodeIsalnum(int c){
    0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
  };

-  if( c<128 ){
-    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
-  }else if( c<(1<<22) ){
+  if( (unsigned int)c<128 ){
+    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
+  }else if( (unsigned int)c<(1<<22) ){
    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
    int iRes = 0;
    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
@@ -322,16 +322,17 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){

  int ret = c;

-  assert( c>=0 );
  assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );

  if( c<128 ){
    if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
  }else if( c<65536 ){
+    const struct TableEntry *p;
    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
    int iLo = 0;
    int iRes = -1;

+    assert( c>aEntry[0].iCode );
    while( iHi>=iLo ){
      int iTest = (iHi + iLo) / 2;
      int cmp = (c - aEntry[iTest].iCode);
@@ -342,14 +343,12 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
        iHi = iTest-1;
      }
    }
-    assert( iRes<0 || c>=aEntry[iRes].iCode );

-    if( iRes>=0 ){
-      const struct TableEntry *p = &aEntry[iRes];
-      if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
-        ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
-        assert( ret>0 );
-      }
+    assert( iRes>=0 && c>=aEntry[iRes].iCode );
+    p = &aEntry[iRes];
+    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
+      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
+      assert( ret>0 );
    }

    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
@@ -407,7 +407,8 @@ static int fts3SqlStmt(
    if( !zSql ){
      rc = SQLITE_NOMEM;
    }else{
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
+      rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
+                              &pStmt, NULL);
      sqlite3_free(zSql);
      assert( rc==SQLITE_OK || pStmt==0 );
      p->aStmt[eStmt] = pStmt;
@@ -4956,11 +4957,14 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
 ** Convert the text beginning at *pz into an integer and return
 ** its value.  Advance *pz to point to the first character past
 ** the integer.
+**
+** This function used for parameters to merge= and incrmerge=
+** commands. 
 */
 static int fts3Getint(const char **pz){
  const char *z = *pz;
  int i = 0;
-  while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0';
+  while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0';
  *pz = z;
  return i;
 }
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+srcdir=`dirname $(dirname $(dirname $(dirname $0)))`
+./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt
+
+echo ""
+
+for f in `ls $srcdir/ext/fts3/*.c` 
+do
+  f=`basename $f`
+  echo -ne "$f: "
+  gcov -b $f | grep Taken | sed 's/Taken at least once://'
+done
+
@@ -227,7 +227,7 @@ proc print_isalnum {zFunc lRange} {
  an_print_ascii_bitmap $lRange
  puts {
  if( (unsigned int)c<128 ){
-    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
+    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
  }else if( (unsigned int)c<(1<<22) ){
    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
    int iRes = 0;
@@ -318,7 +318,7 @@ struct Fts5ExtensionApi {
 ** behaviour. The structure methods are expected to function as follows:
 **
 ** xCreate:
-**   This function is used to allocate and inititalize a tokenizer instance.
+**   This function is used to allocate and initialize a tokenizer instance.
 **   A tokenizer instance is required to actually tokenize text.
 **
 **   The first argument passed to this function is a copy of the (void*)
@@ -577,4 +577,3 @@ struct fts5_api {
 #endif

 #endif /* _FTS5_H */
-
@@ -30,7 +30,9 @@ typedef short i16;
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;

-#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
+#ifndef ArraySize
+# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
+#endif

 #define testcase(x)
 #define ALWAYS(x) 1
@@ -444,9 +446,9 @@ int sqlite3Fts5IndexBeginWrite(

 /*
 ** Flush any data stored in the in-memory hash tables to the database.
-** If the bCommit flag is true, also close any open blob handles.
+** Also close any open blob handles.
 */
-int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit);
+int sqlite3Fts5IndexSync(Fts5Index *p);

 /*
 ** Discard any data stored in the in-memory hash tables. Do not write it
@@ -616,7 +618,7 @@ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);

-int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
+int sqlite3Fts5StorageSync(Fts5Storage *p);
 int sqlite3Fts5StorageRollback(Fts5Storage *p);

 int sqlite3Fts5StorageConfigValue(
@@ -652,6 +654,7 @@ struct Fts5Token {
 /* Parse a MATCH expression. */
 int sqlite3Fts5ExprNew(
  Fts5Config *pConfig, 
+  int iCol,                       /* Column on LHS of MATCH operator */
  const char *zExpr,
  Fts5Expr **ppNew, 
  char **pzErr
@@ -736,7 +739,8 @@ void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
-void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
+void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
+Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);

@@ -189,7 +189,7 @@ static int fts5HighlightCb(
  if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
    p->iOff = iEndOff;
-    if( iPos<p->iter.iEnd ){
+    if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
      fts5HighlightAppend(&rc, p, p->zClose, -1);
    }
  }
@@ -246,6 +246,118 @@ static void fts5HighlightFunction(
 ** End of highlight() implementation.
 **************************************************************************/

+/*
+** Context object passed to the fts5SentenceFinderCb() function.
+*/
+typedef struct Fts5SFinder Fts5SFinder;
+struct Fts5SFinder {
+  int iPos;                       /* Current token position */
+  int nFirstAlloc;                /* Allocated size of aFirst[] */
+  int nFirst;                     /* Number of entries in aFirst[] */
+  int *aFirst;                    /* Array of first token in each sentence */
+  const char *zDoc;               /* Document being tokenized */
+};
+
+/*
+** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
+** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
+** error occurs.
+*/
+static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
+  if( p->nFirstAlloc==p->nFirst ){
+    int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
+    int *aNew;
+
+    aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int));
+    if( aNew==0 ) return SQLITE_NOMEM;
+    p->aFirst = aNew;
+    p->nFirstAlloc = nNew;
+  }
+  p->aFirst[p->nFirst++] = iAdd;
+  return SQLITE_OK;
+}
+
+/*
+** This function is an xTokenize() callback used by the auxiliary snippet()
+** function. Its job is to identify tokens that are the first in a sentence.
+** For each such token, an entry is added to the SFinder.aFirst[] array.
+*/
+static int fts5SentenceFinderCb(
+  void *pContext,                 /* Pointer to HighlightContext object */
+  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStartOff,                  /* Start offset of token */
+  int iEndOff                     /* End offset of token */
+){
+  int rc = SQLITE_OK;
+
+  UNUSED_PARAM2(pToken, nToken);
+  UNUSED_PARAM(iEndOff);
+
+  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
+    Fts5SFinder *p = (Fts5SFinder*)pContext;
+    if( p->iPos>0 ){
+      int i;
+      char c = 0;
+      for(i=iStartOff-1; i>=0; i--){
+        c = p->zDoc[i];
+        if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
+      }
+      if( i!=iStartOff-1 && (c=='.' || c==':') ){
+        rc = fts5SentenceFinderAdd(p, p->iPos);
+      }
+    }else{
+      rc = fts5SentenceFinderAdd(p, 0);
+    }
+    p->iPos++;
+  }
+  return rc;
+}
+
+static int fts5SnippetScore(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  int nDocsize,                   /* Size of column in tokens */
+  unsigned char *aSeen,           /* Array with one element per query phrase */
+  int iCol,                       /* Column to score */
+  int iPos,                       /* Starting offset to score */
+  int nToken,                     /* Max tokens per snippet */
+  int *pnScore,                   /* OUT: Score */
+  int *piPos                      /* OUT: Adjusted offset */
+){
+  int rc;
+  int i;
+  int ip = 0;
+  int ic = 0;
+  int iOff = 0;
+  int iFirst = -1;
+  int nInst;
+  int nScore = 0;
+  int iLast = 0;
+
+  rc = pApi->xInstCount(pFts, &nInst);
+  for(i=0; i<nInst && rc==SQLITE_OK; i++){
+    rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
+    if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){
+      nScore += (aSeen[ip] ? 1 : 1000);
+      aSeen[ip] = 1;
+      if( iFirst<0 ) iFirst = iOff;
+      iLast = iOff + pApi->xPhraseSize(pFts, ip);
+    }
+  }
+
+  *pnScore = nScore;
+  if( piPos ){
+    int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
+    if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
+    if( iAdj<0 ) iAdj = 0;
+    *piPos = iAdj;
+  }
+
+  return rc;
+}
+
 /*
 ** Implementation of snippet() function.
 */
@@ -267,9 +379,10 @@ static void fts5SnippetFunction(
  unsigned char *aSeen;           /* Array of "seen instance" flags */
  int iBestCol;                   /* Column containing best snippet */
  int iBestStart = 0;             /* First token of best snippet */
-  int iBestLast;                  /* Last token of best snippet */
  int nBestScore = 0;             /* Score of best snippet */
  int nColSize = 0;               /* Total size of iBestCol in tokens */
+  Fts5SFinder sFinder;            /* Used to find the beginnings of sentences */
+  int nCol;

  if( nVal!=5 ){
    const char *zErr = "wrong number of arguments to function snippet()";
@@ -277,13 +390,13 @@ static void fts5SnippetFunction(
    return;
  }

+  nCol = pApi->xColumnCount(pFts);
  memset(&ctx, 0, sizeof(HighlightContext));
  iCol = sqlite3_value_int(apVal[0]);
  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
  zEllips = (const char*)sqlite3_value_text(apVal[3]);
  nToken = sqlite3_value_int(apVal[4]);
-  iBestLast = nToken-1;

  iBestCol = (iCol>=0 ? iCol : 0);
  nPhrase = pApi->xPhraseCount(pFts);
@@ -291,65 +404,94 @@ static void fts5SnippetFunction(
  if( aSeen==0 ){
    rc = SQLITE_NOMEM;
  }
-
  if( rc==SQLITE_OK ){
    rc = pApi->xInstCount(pFts, &nInst);
  }
-  for(i=0; rc==SQLITE_OK && i<nInst; i++){
-    int ip, iSnippetCol, iStart;
-    memset(aSeen, 0, nPhrase);
-    rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart);
-    if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){
-      int nScore = 1000;
-      int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip);
-      int j;
-      aSeen[ip] = 1;

-      for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
-        int ic; int io; int iFinal;
-        rc = pApi->xInst(pFts, j, &ip, &ic, &io);
-        iFinal = io + pApi->xPhraseSize(pFts, ip) - 1;
-        if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){
-          nScore += aSeen[ip] ? 1000 : 1;
-          aSeen[ip] = 1;
-          if( iFinal>iLast ) iLast = iFinal;
+  memset(&sFinder, 0, sizeof(Fts5SFinder));
+  for(i=0; i<nCol; i++){
+    if( iCol<0 || iCol==i ){
+      int nDoc;
+      int nDocsize;
+      int ii;
+      sFinder.iPos = 0;
+      sFinder.nFirst = 0;
+      rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xTokenize(pFts, 
+          sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
+      );
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xColumnSize(pFts, i, &nDocsize);
+      if( rc!=SQLITE_OK ) break;
+
+      for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
+        int ip, ic, io;
+        int iAdj;
+        int nScore;
+        int jj;
+
+        rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
+        if( ic!=i || rc!=SQLITE_OK ) continue;
+        memset(aSeen, 0, nPhrase);
+        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
+            io, nToken, &nScore, &iAdj
+        );
+        if( rc==SQLITE_OK && nScore>nBestScore ){
+          nBestScore = nScore;
+          iBestCol = i;
+          iBestStart = iAdj;
+          nColSize = nDocsize;
        }
-      }

-      if( rc==SQLITE_OK && nScore>nBestScore ){
-        iBestCol = iSnippetCol;
-        iBestStart = iStart;
-        iBestLast = iLast;
-        nBestScore = nScore;
+        if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){
+          for(jj=0; jj<(sFinder.nFirst-1); jj++){
+            if( sFinder.aFirst[jj+1]>io ) break;
+          }
+
+          if( sFinder.aFirst[jj]<io ){
+            memset(aSeen, 0, nPhrase);
+            rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
+              sFinder.aFirst[jj], nToken, &nScore, 0
+            );
+
+            nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
+            if( rc==SQLITE_OK && nScore>nBestScore ){
+              nBestScore = nScore;
+              iBestCol = i;
+              iBestStart = sFinder.aFirst[jj];
+              nColSize = nDocsize;
+            }
+          }
+        }
      }
    }
  }

-  if( rc==SQLITE_OK ){
-    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
-  }
  if( rc==SQLITE_OK ){
    rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
  }
+  if( rc==SQLITE_OK && nColSize==0 ){
+    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
+  }
  if( ctx.zIn ){
    if( rc==SQLITE_OK ){
      rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
    }

-    if( (iBestStart+nToken-1)>iBestLast ){
-      iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
-    }
-    if( iBestStart+nToken>nColSize ){
-      iBestStart = nColSize - nToken;
-    }
-    if( iBestStart<0 ) iBestStart = 0;
-
    ctx.iRangeStart = iBestStart;
    ctx.iRangeEnd = iBestStart + nToken - 1;

    if( iBestStart>0 ){
      fts5HighlightAppend(&rc, &ctx, zEllips, -1);
    }
+
+    /* Advance iterator ctx.iter so that it points to the first coalesced
+    ** phrase instance at or following position iBestStart. */
+    while( ctx.iter.iStart>=0 && ctx.iter.iStart<iBestStart && rc==SQLITE_OK ){
+      rc = fts5CInstIterNext(&ctx.iter);
+    }
+
    if( rc==SQLITE_OK ){
      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
    }
@@ -358,15 +500,15 @@ static void fts5SnippetFunction(
    }else{
      fts5HighlightAppend(&rc, &ctx, zEllips, -1);
    }
-
-    if( rc==SQLITE_OK ){
-      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
-    }else{
-      sqlite3_result_error_code(pCtx, rc);
-    }
-    sqlite3_free(ctx.zOut);
  }
+  if( rc==SQLITE_OK ){
+    sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_error_code(pCtx, rc);
+  }
+  sqlite3_free(ctx.zOut);
  sqlite3_free(aSeen);
+  sqlite3_free(sFinder.aFirst);
 }

 /************************************************************************/
@@ -67,9 +67,11 @@ void sqlite3Fts5BufferAppendBlob(
  const u8 *pData
 ){
  assert_nc( *pRc || nData>=0 );
-  if( fts5BufferGrow(pRc, pBuf, nData) ) return;
-  memcpy(&pBuf->p[pBuf->n], pData, nData);
-  pBuf->n += nData;
+  if( nData ){
+    if( fts5BufferGrow(pRc, pBuf, nData) ) return;
+    memcpy(&pBuf->p[pBuf->n], pData, nData);
+    pBuf->n += nData;
+  }
 }

 /*
@@ -246,8 +248,8 @@ void *sqlite3Fts5MallocZero(int *pRc, int nByte){
  void *pRet = 0;
  if( *pRc==SQLITE_OK ){
    pRet = sqlite3_malloc(nByte);
-    if( pRet==0 && nByte>0 ){
-      *pRc = SQLITE_NOMEM;
+    if( pRet==0 ){
+      if( nByte>0 ) *pRc = SQLITE_NOMEM;
    }else{
      memset(pRet, 0, nByte);
    }
@@ -167,6 +167,7 @@ static int fts5ExprGetToken(
    case ',':  tok = FTS5_COMMA; break;
    case '+':  tok = FTS5_PLUS;  break;
    case '*':  tok = FTS5_STAR;  break;
+    case '-':  tok = FTS5_MINUS; break;
    case '\0': tok = FTS5_EOF;   break;

    case '"': {
@@ -212,6 +213,7 @@ static void fts5ParseFree(void *p){ sqlite3_free(p); }

 int sqlite3Fts5ExprNew(
  Fts5Config *pConfig,            /* FTS5 Configuration */
+  int iCol,
  const char *zExpr,              /* Expression text */
  Fts5Expr **ppNew, 
  char **pzErr
@@ -236,6 +238,18 @@ int sqlite3Fts5ExprNew(
  }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
  sqlite3Fts5ParserFree(pEngine, fts5ParseFree);

+  /* If the LHS of the MATCH expression was a user column, apply the
+  ** implicit column-filter.  */
+  if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
+    int n = sizeof(Fts5Colset);
+    Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
+    if( pColset ){
+      pColset->nCol = 1;
+      pColset->aiCol[0] = iCol;
+      sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset);
+    }
+  }
+
  assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
  if( sParse.rc==SQLITE_OK ){
    *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
@@ -745,49 +759,61 @@ static int fts5ExprNearTest(
 ** Initialize all term iterators in the pNear object. If any term is found
 ** to match no documents at all, return immediately without initializing any
 ** further iterators.
+**
+** If an error occurs, return an SQLite error code. Otherwise, return
+** SQLITE_OK. It is not considered an error if some term matches zero
+** documents.
 */
 static int fts5ExprNearInitAll(
  Fts5Expr *pExpr,
  Fts5ExprNode *pNode
 ){
  Fts5ExprNearset *pNear = pNode->pNear;
-  int i, j;
-  int rc = SQLITE_OK;
+  int i;

  assert( pNode->bNomatch==0 );
-  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
+  for(i=0; i<pNear->nPhrase; i++){
    Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
-    for(j=0; j<pPhrase->nTerm; j++){
-      Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
-      Fts5ExprTerm *p;
-      int bEof = 1;
+    if( pPhrase->nTerm==0 ){
+      pNode->bEof = 1;
+      return SQLITE_OK;
+    }else{
+      int j;
+      for(j=0; j<pPhrase->nTerm; j++){
+        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
+        Fts5ExprTerm *p;
+        int bHit = 0;

-      for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){
-        if( p->pIter ){
-          sqlite3Fts5IterClose(p->pIter);
-          p->pIter = 0;
+        for(p=pTerm; p; p=p->pSynonym){
+          int rc;
+          if( p->pIter ){
+            sqlite3Fts5IterClose(p->pIter);
+            p->pIter = 0;
+          }
+          rc = sqlite3Fts5IndexQuery(
+              pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
+              (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
+              (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
+              pNear->pColset,
+              &p->pIter
+          );
+          assert( (rc==SQLITE_OK)==(p->pIter!=0) );
+          if( rc!=SQLITE_OK ) return rc;
+          if( 0==sqlite3Fts5IterEof(p->pIter) ){
+            bHit = 1;
+          }
        }
-        rc = sqlite3Fts5IndexQuery(
-            pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
-            (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
-            (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
-            pNear->pColset,
-            &p->pIter
-        );
-        assert( rc==SQLITE_OK || p->pIter==0 );
-        if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
-          bEof = 0;
-        }
-      }

-      if( bEof ){
-        pNode->bEof = 1;
-        return rc;
+        if( bHit==0 ){
+          pNode->bEof = 1;
+          return SQLITE_OK;
+        }
      }
    }
  }

-  return rc;
+  pNode->bEof = 0;
+  return SQLITE_OK;
 }

 /*
@@ -920,7 +946,7 @@ static int fts5ExprNodeTest_STRING(
          }
        }else{
          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
-          if( pIter->iRowid==iLast ) continue;
+          if( pIter->iRowid==iLast || pIter->bEof ) continue;
          bMatch = 0;
          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
            return rc;
@@ -1097,7 +1123,10 @@ static int fts5ExprNodeNext_OR(
       || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
      ){
        int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
-        if( rc!=SQLITE_OK ) return rc;
+        if( rc!=SQLITE_OK ){
+          pNode->bNomatch = 0;
+          return rc;
+        }
      }
    }
  }
@@ -1128,7 +1157,10 @@ static int fts5ExprNodeTest_AND(
      if( cmp>0 ){
        /* Advance pChild until it points to iLast or laster */
        rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
-        if( rc!=SQLITE_OK ) return rc;
+        if( rc!=SQLITE_OK ){
+          pAnd->bNomatch = 0;
+          return rc;
+        }
      }

      /* If the child node is now at EOF, so is the parent AND node. Otherwise,
@@ -1167,6 +1199,8 @@ static int fts5ExprNodeNext_AND(
  int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
  if( rc==SQLITE_OK ){
    rc = fts5ExprNodeTest_AND(pExpr, pNode);
+  }else{
+    pNode->bNomatch = 0;
  }
  return rc;
 }
@@ -1209,6 +1243,9 @@ static int fts5ExprNodeNext_NOT(
  if( rc==SQLITE_OK ){
    rc = fts5ExprNodeTest_NOT(pExpr, pNode);
  }
+  if( rc!=SQLITE_OK ){
+    pNode->bNomatch = 0;
+  }
  return rc;
 }

@@ -1331,7 +1368,10 @@ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){

  /* If not at EOF but the current rowid occurs earlier than iFirst in
  ** the iteration order, move to document iFirst or later. */
-  if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
+  if( rc==SQLITE_OK 
+   && 0==pRoot->bEof 
+   && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 
+  ){
    rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
  }

@@ -1585,7 +1625,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(

  rc = fts5ParseStringFromToken(pToken, &z);
  if( rc==SQLITE_OK ){
-    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
+    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
    int n;
    sqlite3Fts5Dequote(z);
    n = (int)strlen(z);
@@ -1637,7 +1677,6 @@ int sqlite3Fts5ExprClonePhrase(
 ){
  int rc = SQLITE_OK;             /* Return code */
  Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
-  int i;                          /* Used to iterate through phrase terms */
  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
  TokenCtx sCtx = {0,0};          /* Context object for fts5ParseTokenize */

@@ -1658,7 +1697,7 @@ int sqlite3Fts5ExprClonePhrase(
  if( rc==SQLITE_OK ){
    Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset;
    if( pColsetOrig ){
-      int nByte = sizeof(Fts5Colset) + pColsetOrig->nCol * sizeof(int);
+      int nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int);
      Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte);
      if( pColset ){ 
        memcpy(pColset, pColsetOrig, nByte);
@@ -1667,18 +1706,25 @@ int sqlite3Fts5ExprClonePhrase(
    }
  }

-  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
-    int tflags = 0;
-    Fts5ExprTerm *p;
-    for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
-      const char *zTerm = p->zTerm;
-      rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
-          0, 0);
-      tflags = FTS5_TOKEN_COLOCATED;
-    }
-    if( rc==SQLITE_OK ){
-      sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+  if( pOrig->nTerm ){
+    int i;                          /* Used to iterate through phrase terms */
+    for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
+      int tflags = 0;
+      Fts5ExprTerm *p;
+      for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
+        const char *zTerm = p->zTerm;
+        rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
+            0, 0);
+        tflags = FTS5_TOKEN_COLOCATED;
+      }
+      if( rc==SQLITE_OK ){
+        sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+      }
    }
+  }else{
+    /* This happens when parsing a token or quoted phrase that contains
+    ** no token characters at all. (e.g ... MATCH '""'). */
+    sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
  }

  if( rc==SQLITE_OK ){
@@ -1793,6 +1839,34 @@ static Fts5Colset *fts5ParseColset(
  return pNew;
 }

+/*
+** Allocate and return an Fts5Colset object specifying the inverse of
+** the colset passed as the second argument. Free the colset passed
+** as the second argument before returning.
+*/
+Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){
+  Fts5Colset *pRet;
+  int nCol = pParse->pConfig->nCol;
+
+  pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, 
+      sizeof(Fts5Colset) + sizeof(int)*nCol
+  );
+  if( pRet ){
+    int i;
+    int iOld = 0;
+    for(i=0; i<nCol; i++){
+      if( iOld>=p->nCol || p->aiCol[iOld]!=i ){
+        pRet->aiCol[pRet->nCol++] = i;
+      }else{
+        iOld++;
+      }
+    }
+  }
+
+  sqlite3_free(p);
+  return pRet;
+}
+
 Fts5Colset *sqlite3Fts5ParseColset(
  Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  Fts5Colset *pColset,            /* Existing colset object */
@@ -1825,25 +1899,110 @@ Fts5Colset *sqlite3Fts5ParseColset(
  return pRet;
 }

+/*
+** If argument pOrig is NULL, or if (*pRc) is set to anything other than
+** SQLITE_OK when this function is called, NULL is returned. 
+**
+** Otherwise, a copy of (*pOrig) is made into memory obtained from
+** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
+** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
+*/
+static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
+  Fts5Colset *pRet;
+  if( pOrig ){
+    int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
+    pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
+    if( pRet ){ 
+      memcpy(pRet, pOrig, nByte);
+    }
+  }else{
+    pRet = 0;
+  }
+  return pRet;
+}
+
+/*
+** Remove from colset pColset any columns that are not also in colset pMerge.
+*/
+static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
+  int iIn = 0;          /* Next input in pColset */
+  int iMerge = 0;       /* Next input in pMerge */
+  int iOut = 0;         /* Next output slot in pColset */
+
+  while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
+    int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
+    if( iDiff==0 ){
+      pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
+      iMerge++;
+      iIn++;
+    }else if( iDiff>0 ){
+      iMerge++;
+    }else{
+      iIn++;
+    }
+  }
+  pColset->nCol = iOut;
+}
+
+/*
+** Recursively apply colset pColset to expression node pNode and all of
+** its decendents. If (*ppFree) is not NULL, it contains a spare copy
+** of pColset. This function may use the spare copy and set (*ppFree) to
+** zero, or it may create copies of pColset using fts5CloneColset().
+*/
+static void fts5ParseSetColset(
+  Fts5Parse *pParse, 
+  Fts5ExprNode *pNode, 
+  Fts5Colset *pColset,
+  Fts5Colset **ppFree
+){
+  if( pParse->rc==SQLITE_OK ){
+    assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING 
+         || pNode->eType==FTS5_AND  || pNode->eType==FTS5_OR
+         || pNode->eType==FTS5_NOT  || pNode->eType==FTS5_EOF
+    );
+    if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
+      Fts5ExprNearset *pNear = pNode->pNear;
+      if( pNear->pColset ){
+        fts5MergeColset(pNear->pColset, pColset);
+        if( pNear->pColset->nCol==0 ){
+          pNode->eType = FTS5_EOF;
+          pNode->xNext = 0;
+        }
+      }else if( *ppFree ){
+        pNear->pColset = pColset;
+        *ppFree = 0;
+      }else{
+        pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
+      }
+    }else{
+      int i;
+      assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
+      for(i=0; i<pNode->nChild; i++){
+        fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
+      }
+    }
+  }
+}
+
+/*
+** Apply colset pColset to expression node pExpr and all of its descendents.
+*/
 void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
-  Fts5ExprNearset *pNear, 
+  Fts5ExprNode *pExpr, 
  Fts5Colset *pColset 
 ){
+  Fts5Colset *pFree = pColset;
  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
    pParse->rc = SQLITE_ERROR;
    pParse->zErr = sqlite3_mprintf(
      "fts5: column queries are not supported (detail=none)"
    );
-    sqlite3_free(pColset);
-    return;
-  }
-
-  if( pNear ){
-    pNear->pColset = pColset;
  }else{
-    sqlite3_free(pColset);
+    fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
  }
+  sqlite3_free(pFree);
 }

 static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
@@ -2297,7 +2456,7 @@ static void fts5ExprFunction(

  rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  if( rc==SQLITE_OK ){
-    rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
+    rc = sqlite3Fts5ExprNew(pConfig, pConfig->nCol, zExpr, &pExpr, &zErr);
  }
  if( rc==SQLITE_OK ){
    char *zText;
@@ -36,9 +36,10 @@ struct Fts5Hash {

 /*
 ** Each entry in the hash table is represented by an object of the 
-** following type. Each object, its key (zKey[]) and its current data
-** are stored in a single memory allocation. The position list data 
-** immediately follows the key data in memory.
+** following type. Each object, its key (a nul-terminated string) and 
+** its current data are stored in a single memory allocation. The 
+** key immediately follows the object in memory. The position list
+** data immediately follows the key data in memory.
 **
 ** The data that follows the key is in a similar, but not identical format
 ** to the doclist data stored in the database. It is:
@@ -62,20 +63,20 @@ struct Fts5HashEntry {
  int nAlloc;                     /* Total size of allocation */
  int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
  int nData;                      /* Total bytes of data (incl. structure) */
-  int nKey;                       /* Length of zKey[] in bytes */
+  int nKey;                       /* Length of key in bytes */
  u8 bDel;                        /* Set delete-flag @ iSzPoslist */
  u8 bContent;                    /* Set content-flag (detail=none mode) */
  i16 iCol;                       /* Column of last value written */
  int iPos;                       /* Position of last value written */
  i64 iRowid;                     /* Rowid of last value written */
-  char zKey[8];                   /* Nul-terminated entry key */
 };

 /*
-** Size of Fts5HashEntry without the zKey[] array.
+** Eqivalent to:
+**
+**   char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; }
 */
-#define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8)
-
+#define fts5EntryKey(p) ( ((char *)(&(p)[1])) )


 /*
@@ -170,10 +171,11 @@ static int fts5HashResize(Fts5Hash *pHash){

  for(i=0; i<pHash->nSlot; i++){
    while( apOld[i] ){
-      int iHash;
+      unsigned int iHash;
      Fts5HashEntry *p = apOld[i];
      apOld[i] = p->pHashNext;
-      iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey));
+      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
+                          (int)strlen(fts5EntryKey(p)));
      p->pHashNext = apNew[iHash];
      apNew[iHash] = p;
    }
@@ -244,9 +246,10 @@ int sqlite3Fts5HashWrite(
  /* Attempt to locate an existing hash entry */
  iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
-    if( p->zKey[0]==bByte 
+    char *zKey = fts5EntryKey(p);
+    if( zKey[0]==bByte 
     && p->nKey==nToken
-     && memcmp(&p->zKey[1], pToken, nToken)==0 
+     && memcmp(&zKey[1], pToken, nToken)==0 
    ){
      break;
    }
@@ -255,7 +258,8 @@ int sqlite3Fts5HashWrite(
  /* If an existing hash entry cannot be found, create a new one. */
  if( p==0 ){
    /* Figure out how much space to allocate */
-    int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
+    char *zKey;
+    int nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64;
    if( nByte<128 ) nByte = 128;

    /* Grow the Fts5Hash.aSlot[] array if necessary. */
@@ -268,14 +272,15 @@ int sqlite3Fts5HashWrite(
    /* Allocate new Fts5HashEntry and add it to the hash table. */
    p = (Fts5HashEntry*)sqlite3_malloc(nByte);
    if( !p ) return SQLITE_NOMEM;
-    memset(p, 0, FTS5_HASHENTRYSIZE);
+    memset(p, 0, sizeof(Fts5HashEntry));
    p->nAlloc = nByte;
-    p->zKey[0] = bByte;
-    memcpy(&p->zKey[1], pToken, nToken);
-    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
+    zKey = fts5EntryKey(p);
+    zKey[0] = bByte;
+    memcpy(&zKey[1], pToken, nToken);
+    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
    p->nKey = nToken;
-    p->zKey[nToken+1] = '\0';
-    p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
+    zKey[nToken+1] = '\0';
+    p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
    p->pHashNext = pHash->aSlot[iHash];
    pHash->aSlot[iHash] = p;
    pHash->nEntry++;
@@ -393,9 +398,11 @@ static Fts5HashEntry *fts5HashEntryMerge(
      p1 = 0;
    }else{
      int i = 0;
-      while( p1->zKey[i]==p2->zKey[i] ) i++;
+      char *zKey1 = fts5EntryKey(p1);
+      char *zKey2 = fts5EntryKey(p2);
+      while( zKey1[i]==zKey2[i] ) i++;

-      if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){
+      if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
        /* p2 is smaller */
        *ppOut = p2;
        ppOut = &p2->pScanNext;
@@ -438,7 +445,7 @@ static int fts5HashEntrySort(
  for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
    Fts5HashEntry *pIter;
    for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
-      if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){
+      if( pTerm==0 || 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm) ){
        Fts5HashEntry *pEntry = pIter;
        pEntry->pScanNext = 0;
        for(i=0; ap[i]; i++){
@@ -471,16 +478,18 @@ int sqlite3Fts5HashQuery(
  int *pnDoclist                  /* OUT: Size of doclist in bytes */
 ){
  unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
+  char *zKey = 0;
  Fts5HashEntry *p;

  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
-    if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break;
+    zKey = fts5EntryKey(p);
+    if( memcmp(zKey, pTerm, nTerm)==0 && zKey[nTerm]==0 ) break;
  }

  if( p ){
    fts5HashAddPoslistSize(pHash, p);
-    *ppDoclist = (const u8*)&p->zKey[nTerm+1];
-    *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
+    *ppDoclist = (const u8*)&zKey[nTerm+1];
+    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
  }else{
    *ppDoclist = 0;
    *pnDoclist = 0;
@@ -513,11 +522,12 @@ void sqlite3Fts5HashScanEntry(
 ){
  Fts5HashEntry *p;
  if( (p = pHash->pScan) ){
-    int nTerm = (int)strlen(p->zKey);
+    char *zKey = fts5EntryKey(p);
+    int nTerm = (int)strlen(zKey);
    fts5HashAddPoslistSize(pHash, p);
-    *pzTerm = p->zKey;
-    *ppDoclist = (const u8*)&p->zKey[nTerm+1];
-    *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
+    *pzTerm = zKey;
+    *ppDoclist = (const u8*)&zKey[nTerm+1];
+    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
  }else{
    *pzTerm = 0;
    *ppDoclist = 0;
@@ -628,7 +628,6 @@ static void fts5CloseReader(Fts5Index *p){
  }
 }

-
 /*
 ** Retrieve a record from the %_data table.
 **
@@ -702,7 +701,6 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
  return pRet;
 }

-
 /*
 ** Release a reference to data record returned by an earlier call to
 ** fts5DataRead().
@@ -711,6 +709,18 @@ static void fts5DataRelease(Fts5Data *pData){
  sqlite3_free(pData);
 }

+static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){
+  Fts5Data *pRet = fts5DataRead(p, iRowid);
+  if( pRet ){
+    if( pRet->szLeaf>pRet->nn ){
+      p->rc = FTS5_CORRUPT;
+      fts5DataRelease(pRet);
+      pRet = 0;
+    }
+  }
+  return pRet;
+}
+
 static int fts5IndexPrepareStmt(
  Fts5Index *p,
  sqlite3_stmt **ppStmt,
@@ -718,7 +728,8 @@ static int fts5IndexPrepareStmt(
 ){
  if( p->rc==SQLITE_OK ){
    if( zSql ){
-      p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0);
+      p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1,
+                                 SQLITE_PREPARE_PERSISTENT, ppStmt, 0);
    }else{
      p->rc = SQLITE_NOMEM;
    }
@@ -767,7 +778,8 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){
    if( zSql==0 ){
      rc = SQLITE_NOMEM;
    }else{
-      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0);
+      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
+                              SQLITE_PREPARE_PERSISTENT, &p->pDeleter, 0);
      sqlite3_free(zSql);
    }
    if( rc!=SQLITE_OK ){
@@ -1519,7 +1531,7 @@ static void fts5SegIterNextPage(
    pIter->pLeaf = pIter->pNextLeaf;
    pIter->pNextLeaf = 0;
  }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
-    pIter->pLeaf = fts5DataRead(p, 
+    pIter->pLeaf = fts5LeafRead(p, 
        FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno)
    );
  }else{
@@ -2022,14 +2034,13 @@ static void fts5SegIterNext(
        if( pLeaf->nn>pLeaf->szLeaf ){
          pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
              &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
-              );
+          );
        }
-
      }
      else if( pLeaf->nn>pLeaf->szLeaf ){
        pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
            &pLeaf->p[pLeaf->szLeaf], iOff
-            );
+        );
        pIter->iLeafOffset = iOff;
        pIter->iEndofDoclist = iOff;
        bNewTerm = 1;
@@ -2063,6 +2074,7 @@ static void fts5SegIterNext(
      */
      int nSz;
      assert( p->rc==SQLITE_OK );
+      assert( pIter->iLeafOffset<=pIter->pLeaf->nn );
      fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
      pIter->bDel = (nSz & 0x0001);
      pIter->nPos = nSz>>1;
@@ -2269,6 +2281,11 @@ static void fts5LeafSeek(
    iTermOff += nKeep;
    iOff = iTermOff;

+    if( iOff>=n ){
+      p->rc = FTS5_CORRUPT;
+      return;
+    }
+
    /* Read the nKeep field of the next term. */
    fts5FastGetVarint32(a, iOff, nKeep);
  }
@@ -2825,6 +2842,7 @@ static void fts5MultiIterNext(
  i64 iFrom                       /* Advance at least as far as this */
 ){
  int bUseFrom = bFrom;
+  assert( pIter->base.bEof==0 );
  while( p->rc==SQLITE_OK ){
    int iFirst = pIter->aFirst[1].iFirst;
    int bNewTerm = 0;
@@ -2862,7 +2880,8 @@ static void fts5MultiIterNext2(
 ){
  assert( pIter->bSkipEmpty );
  if( p->rc==SQLITE_OK ){
-    do {
+    *pbNewTerm = 0;
+    do{
      int iFirst = pIter->aFirst[1].iFirst;
      Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
      int bNewTerm = 0;
@@ -2875,8 +2894,6 @@ static void fts5MultiIterNext2(
        fts5MultiIterAdvanced(p, pIter, iFirst, 1);
        fts5MultiIterSetEof(pIter);
        *pbNewTerm = 1;
-      }else{
-        *pbNewTerm = 0;
      }
      fts5AssertMultiIterSetup(p, pIter);

@@ -3051,7 +3068,7 @@ static void fts5ChunkIterate(
      break;
    }else{
      pgno++;
-      pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno));
+      pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno));
      if( pData==0 ) break;
      pChunk = &pData->p[4];
      nChunk = MIN(nRem, pData->szLeaf - 4);
@@ -3142,23 +3159,23 @@ static int fts5IndexExtractCol(
  return p - (*pa);
 }

-static int fts5IndexExtractColset (
+static void fts5IndexExtractColset(
+  int *pRc,
  Fts5Colset *pColset,            /* Colset to filter on */
  const u8 *pPos, int nPos,       /* Position list */
  Fts5Buffer *pBuf                /* Output buffer */
 ){
-  int rc = SQLITE_OK;
-  int i;
-
-  fts5BufferZero(pBuf);
-  for(i=0; i<pColset->nCol; i++){
-    const u8 *pSub = pPos;
-    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
-    if( nSub ){
-      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
+  if( *pRc==SQLITE_OK ){
+    int i;
+    fts5BufferZero(pBuf);
+    for(i=0; i<pColset->nCol; i++){
+      const u8 *pSub = pPos;
+      int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
+      if( nSub ){
+        fts5BufferAppendBlob(pRc, pBuf, nSub, pSub);
+      }
    }
  }
-  return rc;
 }

 /*
@@ -3195,6 +3212,15 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
  }
 }

+/*
+** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match
+** against no columns at all).
+*/
+static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){
+  UNUSED_PARAM(pSeg);
+  pIter->base.nData = 0;
+}
+
 /*
 ** xSetOutputs callback used by detail=col when there is a column filter
 ** and there are 100 or more columns. Also called as a fallback from
@@ -3273,8 +3299,9 @@ static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
      pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
      pIter->base.pData = a;
    }else{
+      int *pRc = &pIter->pIndex->rc;
      fts5BufferZero(&pIter->poslist);
-      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
+      fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist);
      pIter->base.pData = pIter->poslist.p;
      pIter->base.nData = pIter->poslist.n;
    }
@@ -3300,6 +3327,10 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
      pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
    }

+    else if( pIter->pColset->nCol==0 ){
+      pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset;
+    }
+
    else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
      pIter->xSetOutputs = fts5IterSetOutputs_Full;
    }
@@ -3815,9 +3846,6 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
  Fts5PageWriter *pPage = &pWriter->writer;
  i64 iRowid;

-static int nCall = 0;
-nCall++;
-
  assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) );

  /* Set the szLeaf header field. */
@@ -4166,6 +4194,7 @@ static void fts5IndexMergeLevel(
  int bOldest;                    /* True if the output segment is the oldest */
  int eDetail = p->pConfig->eDetail;
  const int flags = FTS5INDEX_QUERY_NOOUTPUT;
+  int bTermWritten = 0;           /* True if current term already output */

  assert( iLvl<pStruct->nLevel );
  assert( pLvl->nMerge<=pLvl->nSeg );
@@ -4219,18 +4248,22 @@ static void fts5IndexMergeLevel(
    int nTerm;
    const u8 *pTerm;

-    /* Check for key annihilation. */
-    if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue;
-
    pTerm = fts5MultiIterTerm(pIter, &nTerm);
    if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){
      if( pnRem && writer.nLeafWritten>nRem ){
        break;
      }
+      fts5BufferSet(&p->rc, &term, nTerm, pTerm);
+      bTermWritten =0;
+    }

+    /* Check for key annihilation. */
+    if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue;
+
+    if( p->rc==SQLITE_OK && bTermWritten==0 ){
      /* This is a new term. Append a term to the output segment. */
      fts5WriteAppendTerm(p, &writer, nTerm, pTerm);
-      fts5BufferSet(&p->rc, &term, nTerm, pTerm);
+      bTermWritten = 1;
    }

    /* Append the rowid to the output */
@@ -5062,7 +5095,7 @@ static void fts5SetupPrefixIter(
    if( pData ){
      pData->p = (u8*)&pData[1];
      pData->nn = pData->szLeaf = doclist.n;
-      memcpy(pData->p, doclist.p, doclist.n);
+      if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n);
      fts5MultiIterNew2(p, pData, bDesc, ppIter);
    }
    fts5BufferFree(&doclist);
@@ -5101,10 +5134,10 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
 /*
 ** Commit data to disk.
 */
-int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
+int sqlite3Fts5IndexSync(Fts5Index *p){
  assert( p->rc==SQLITE_OK );
  fts5IndexFlush(p);
-  if( bCommit ) fts5CloseReader(p);
+  fts5CloseReader(p);
  return fts5IndexReturn(p);
 }

@@ -5301,7 +5334,7 @@ int sqlite3Fts5IndexQuery(

  if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
    int iIdx = 0;                 /* Index to search */
-    memcpy(&buf.p[1], pToken, nToken);
+    if( nToken ) memcpy(&buf.p[1], pToken, nToken);

    /* Figure out which index to search and set iIdx accordingly. If this
    ** is a prefix query for which there is no prefix index, set iIdx to
@@ -5350,7 +5383,7 @@ int sqlite3Fts5IndexQuery(
    }

    if( p->rc ){
-      sqlite3Fts5IterClose(&pRet->base);
+      sqlite3Fts5IterClose((Fts5IndexIter*)pRet);
      pRet = 0;
      fts5CloseReader(p);
    }
@@ -5800,7 +5833,7 @@ static void fts5IndexIntegrityCheckSegment(
    ** ignore this b-tree entry. Otherwise, load it into memory. */
    if( iIdxLeaf<pSeg->pgnoFirst ) continue;
    iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf);
-    pLeaf = fts5DataRead(p, iRow);
+    pLeaf = fts5LeafRead(p, iRow);
    if( pLeaf==0 ) break;

    /* Check that the leaf contains at least one term, and that it is equal
@@ -506,6 +506,7 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
 static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  Fts5Table *pTab = (Fts5Table*)pVTab;
  Fts5Config *pConfig = pTab->pConfig;
+  const int nCol = pConfig->nCol;
  int idxFlags = 0;               /* Parameter passed through to xFilter() */
  int bHasMatch;
  int iNext;
@@ -531,24 +532,34 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){

  int aColMap[3];
  aColMap[0] = -1;
-  aColMap[1] = pConfig->nCol;
-  aColMap[2] = pConfig->nCol+1;
+  aColMap[1] = nCol;
+  aColMap[2] = nCol+1;

  /* Set idxFlags flags for all WHERE clause terms that will be used. */
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
-    int j;
-    for(j=0; j<ArraySize(aConstraint); j++){
-      struct Constraint *pC = &aConstraint[j];
-      if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){
-        if( p->usable ){
+    int iCol = p->iColumn;
+
+    if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol)
+     || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol==nCol)
+    ){
+      /* A MATCH operator or equivalent */
+      if( p->usable ){
+        idxFlags = (idxFlags & 0xFFFF) | FTS5_BI_MATCH | (iCol << 16);
+        aConstraint[0].iConsIndex = i;
+      }else{
+        /* As there exists an unusable MATCH constraint this is an 
+        ** unusable plan. Set a prohibitively high cost. */
+        pInfo->estimatedCost = 1e50;
+        return SQLITE_OK;
+      }
+    }else{
+      int j;
+      for(j=1; j<ArraySize(aConstraint); j++){
+        struct Constraint *pC = &aConstraint[j];
+        if( iCol==aColMap[pC->iCol] && p->op & pC->op && p->usable ){
          pC->iConsIndex = i;
          idxFlags |= pC->fts5op;
-        }else if( j==0 ){
-          /* As there exists an unusable MATCH constraint this is an 
-          ** unusable plan. Set a prohibitively high cost. */
-          pInfo->estimatedCost = 1e50;
-          return SQLITE_OK;
        }
      }
    }
@@ -872,7 +883,8 @@ static int fts5PrepareStatement(
  if( zSql==0 ){
    rc = SQLITE_NOMEM; 
  }else{
-    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
+    rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, 
+                            SQLITE_PREPARE_PERSISTENT, &pRet, 0);
    if( rc!=SQLITE_OK ){
      *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
    }
@@ -1008,7 +1020,8 @@ static int fts5FindRankFunction(Fts5Cursor *pCsr){
    char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs);
    if( zSql ){
      sqlite3_stmt *pStmt = 0;
-      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0);
+      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
+                              SQLITE_PREPARE_PERSISTENT, &pStmt, 0);
      sqlite3_free(zSql);
      assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 );
      if( rc==SQLITE_OK ){
@@ -1123,6 +1136,7 @@ static int fts5FilterMethod(
  sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
+  int iCol;                       /* Column on LHS of MATCH operator */
  char **pzErrmsg = pConfig->pzErrmsg;

  UNUSED_PARAM(zUnused);
@@ -1153,6 +1167,8 @@ static int fts5FilterMethod(
  if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++];
  if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++];
  if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++];
+  iCol = (idxNum>>16);
+  assert( iCol>=0 && iCol<=pConfig->nCol );
  assert( iVal==nVal );
  bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0);
  pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0);
@@ -1199,7 +1215,7 @@ static int fts5FilterMethod(
        rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
      }else{
        char **pzErr = &pTab->base.zErrMsg;
-        rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr);
+        rc = sqlite3Fts5ExprNew(pConfig, iCol, zExpr, &pCsr->pExpr, pzErr);
        if( rc==SQLITE_OK ){
          if( bOrderByRank ){
            pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
@@ -1579,7 +1595,7 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
  fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
  pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;
  fts5TripCursors(pTab);
-  rc = sqlite3Fts5StorageSync(pTab->pStorage, 1);
+  rc = sqlite3Fts5StorageSync(pTab->pStorage);
  pTab->pConfig->pzErrmsg = 0;
  return rc;
 }
@@ -2390,7 +2406,7 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
  fts5TripCursors(pTab);
-  return sqlite3Fts5StorageSync(pTab->pStorage, 0);
+  return sqlite3Fts5StorageSync(pTab->pStorage);
 }

 /*
@@ -2403,7 +2419,7 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
  fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
  fts5TripCursors(pTab);
-  return sqlite3Fts5StorageSync(pTab->pStorage, 0);
+  return sqlite3Fts5StorageSync(pTab->pStorage);
 }

 /*
@@ -2593,15 +2609,14 @@ static void fts5ModuleDestroy(void *pCtx){
 static void fts5Fts5Func(
  sqlite3_context *pCtx,          /* Function call context */
  int nArg,                       /* Number of args */
-  sqlite3_value **apUnused        /* Function arguments */
+  sqlite3_value **apArg           /* Function arguments */
 ){
  Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
-  char buf[8];
-  UNUSED_PARAM2(nArg, apUnused);
-  assert( nArg==0 );
-  assert( sizeof(buf)>=sizeof(pGlobal) );
-  memcpy(buf, (void*)&pGlobal, sizeof(pGlobal));
-  sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
+  fts5_api **ppApi;
+  UNUSED_PARAM(nArg);
+  assert( nArg==1 );
+  ppApi = (fts5_api**)sqlite3_value_pointer(apArg[0], "fts5_api_ptr");
+  if( ppApi ) *ppApi = &pGlobal->api;
 }

 /*
@@ -2666,7 +2681,7 @@ static int fts5Init(sqlite3 *db){
    if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db);
    if( rc==SQLITE_OK ){
      rc = sqlite3_create_function(
-          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
+          db, "fts5", 1, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
      );
    }
    if( rc==SQLITE_OK ){
@@ -136,7 +136,8 @@ static int fts5StorageGetStmt(
    if( zSql==0 ){
      rc = SQLITE_NOMEM;
    }else{
-      rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
+      rc = sqlite3_prepare_v3(pC->db, zSql, -1,
+                              SQLITE_PREPARE_PERSISTENT, &p->aStmt[eStmt], 0);
      sqlite3_free(zSql);
      if( rc!=SQLITE_OK && pzErrMsg ){
        *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
@@ -218,7 +219,7 @@ static void fts5StorageRenameOne(

 int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
  Fts5Config *pConfig = pStorage->pConfig;
-  int rc = sqlite3Fts5StorageSync(pStorage, 1);
+  int rc = sqlite3Fts5StorageSync(pStorage);

  fts5StorageRenameOne(pConfig, &rc, "data", zName);
  fts5StorageRenameOne(pConfig, &rc, "idx", zName);
@@ -545,11 +546,6 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
    }
  }

-  /* Write the averages record */
-  if( rc==SQLITE_OK ){
-    rc = fts5StorageSaveTotals(p);
-  }
-
  return rc;
 }

@@ -753,11 +749,6 @@ int sqlite3Fts5StorageIndexInsert(
  }
  sqlite3_free(buf.p);

-  /* Write the averages record */
-  if( rc==SQLITE_OK ){
-    rc = fts5StorageSaveTotals(p);
-  }
-
  return rc;
 }

@@ -1091,13 +1082,18 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
 /*
 ** Flush any data currently held in-memory to disk.
 */
-int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){
-  if( bCommit && p->bTotalsValid ){
-    int rc = fts5StorageSaveTotals(p);
+int sqlite3Fts5StorageSync(Fts5Storage *p){
+  int rc = SQLITE_OK;
+  i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
+  if( p->bTotalsValid ){
+    rc = fts5StorageSaveTotals(p);
    p->bTotalsValid = 0;
-    if( rc!=SQLITE_OK ) return rc;
  }
-  return sqlite3Fts5IndexSync(p->pIndex, bCommit);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexSync(p->pIndex);
+  }
+  sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid);
+  return rc;
 }

 int sqlite3Fts5StorageRollback(Fts5Storage *p){
@@ -14,7 +14,14 @@


 #ifdef SQLITE_TEST
-#include <tcl.h>
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif

 #ifdef SQLITE_ENABLE_FTS5

@@ -78,7 +85,7 @@ static int f5tResultToErrorCode(const char *zRes){
  return SQLITE_ERROR;
 }

-static int f5tDbAndApi(
+static int SQLITE_TCLAPI f5tDbAndApi(
  Tcl_Interp *interp, 
  Tcl_Obj *pObj, 
  sqlite3 **ppDb, 
@@ -92,16 +99,13 @@ static int f5tDbAndApi(
    sqlite3_stmt *pStmt = 0;
    fts5_api *pApi = 0;

-    rc = sqlite3_prepare_v2(db, "SELECT fts5()", -1, &pStmt, 0);
+    rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);
    if( rc!=SQLITE_OK ){
      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
      return TCL_ERROR;
    }
-
-    if( SQLITE_ROW==sqlite3_step(pStmt) ){
-      const void *pPtr = sqlite3_column_blob(pStmt, 0);
-      memcpy((void*)&pApi, pPtr, sizeof(pApi));
-    }
+    sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0);
+    sqlite3_step(pStmt);

    if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
@@ -164,7 +168,7 @@ static int xTokenizeCb(
  return rc;
 }

-static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);
+static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);

 static int xQueryPhraseCb(
  const Fts5ExtensionApi *pApi, 
@@ -209,7 +213,7 @@ static void xSetAuxdataDestructor(void *p){
 **
 ** Description...
 */
-static int xF5tApi(
+static int SQLITE_TCLAPI xF5tApi(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -602,7 +606,7 @@ static void xF5tDestroy(void *pCtx){
 **
 ** Description...
 */
-static int f5tCreateFunction(
+static int SQLITE_TCLAPI f5tCreateFunction(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -672,7 +676,7 @@ static int xTokenizeCb2(
 **
 ** Description...
 */
-static int f5tTokenize(
+static int SQLITE_TCLAPI f5tTokenize(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -878,7 +882,7 @@ static int f5tTokenizerTokenize(
 /*
 ** sqlite3_fts5_token ?-colocated? TEXT START END
 */
-static int f5tTokenizerReturn(
+static int SQLITE_TCLAPI f5tTokenizerReturn(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -949,7 +953,7 @@ static void f5tDelTokenizer(void *pCtx){
 ** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each
 ** token within the tokenized text.
 */
-static int f5tCreateTokenizer(
+static int SQLITE_TCLAPI f5tCreateTokenizer(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
@@ -992,7 +996,7 @@ static int f5tCreateTokenizer(
  return TCL_OK;
 }

-static void xF5tFree(ClientData clientData){
+static void SQLITE_TCLAPI xF5tFree(ClientData clientData){
  ckfree(clientData);
 }

@@ -1001,7 +1005,7 @@ static void xF5tFree(ClientData clientData){
 **
 ** Set or clear the global "may-be-corrupt" flag. Return the old value.
 */
-static int f5tMayBeCorrupt(
+static int SQLITE_TCLAPI f5tMayBeCorrupt(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -1033,7 +1037,7 @@ static unsigned int f5t_fts5HashKey(int nSlot, const char *p, int n){
  return (h % nSlot);
 }

-static int f5tTokenHash(
+static int SQLITE_TCLAPI f5tTokenHash(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -1058,7 +1062,7 @@ static int f5tTokenHash(
  return TCL_OK;
 }

-static int f5tRegisterMatchinfo(
+static int SQLITE_TCLAPI f5tRegisterMatchinfo(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -1083,7 +1087,7 @@ static int f5tRegisterMatchinfo(
  return TCL_OK;
 }

-static int f5tRegisterTok(
+static int SQLITE_TCLAPI f5tRegisterTok(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -73,13 +73,10 @@ static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
  int rc;

  *ppApi = 0;
-  rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0);
+  rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0);
  if( rc==SQLITE_OK ){
-    if( SQLITE_ROW==sqlite3_step(pStmt) 
-        && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0)
-      ){
-      memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*));
-    }
+    sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0);
+    (void)sqlite3_step(pStmt);
    rc = sqlite3_finalize(pStmt);
  }

@@ -422,4 +419,3 @@ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){
 }

 #endif /* SQLITE_ENABLE_FTS5 */
-
@@ -40,7 +40,7 @@
 */
 #if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)

-#include <fts5.h>
+#include "fts5.h"
 #include <string.h>
 #include <assert.h>

@@ -182,7 +182,7 @@ static int fts5tokConnectMethod(
  Fts5tokTable *pTab = 0;
  int rc;
  char **azDequote = 0;
-  int nDequote;
+  int nDequote = 0;

  rc = sqlite3_declare_vtab(db, 
       "CREATE TABLE x(input HIDDEN, token, start, end, position)"
@@ -29,6 +29,11 @@
 **   the number of fts5 rows that contain at least one instance of term
 **   $term. Field $cnt is set to the total number of instances of term 
 **   $term in the database.
+**
+** instance:
+**     CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY(<all-fields>));
+**
+**   One row for each term instance in the database. 
 */


@@ -44,7 +49,7 @@ struct Fts5VocabTable {
  char *zFts5Db;                  /* Db containing fts5 table */
  sqlite3 *db;                    /* Database handle */
  Fts5Global *pGlobal;            /* FTS5 global object for this database */
-  int eType;                      /* FTS5_VOCAB_COL or ROW */
+  int eType;                      /* FTS5_VOCAB_COL, ROW or INSTANCE */
 };

 struct Fts5VocabCursor {
@@ -64,16 +69,22 @@ struct Fts5VocabCursor {
  i64 *aCnt;
  i64 *aDoc;

-  /* Output values used by 'row' and 'col' tables */
+  /* Output values used by all tables. */
  i64 rowid;                      /* This table's current rowid value */
  Fts5Buffer term;                /* Current value of 'term' column */
+
+  /* Output values Used by 'instance' tables only */
+  i64 iInstPos;
+  int iInstOff;
 };

-#define FTS5_VOCAB_COL    0
-#define FTS5_VOCAB_ROW    1
+#define FTS5_VOCAB_COL      0
+#define FTS5_VOCAB_ROW      1
+#define FTS5_VOCAB_INSTANCE 2

 #define FTS5_VOCAB_COL_SCHEMA  "term, col, doc, cnt"
 #define FTS5_VOCAB_ROW_SCHEMA  "term, doc, cnt"
+#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset"

 /*
 ** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
@@ -101,6 +112,9 @@ static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
    if( sqlite3_stricmp(zCopy, "row")==0 ){
      *peType = FTS5_VOCAB_ROW;
    }else
+    if( sqlite3_stricmp(zCopy, "instance")==0 ){
+      *peType = FTS5_VOCAB_INSTANCE;
+    }else
    {
      *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy);
      rc = SQLITE_ERROR;
@@ -161,7 +175,8 @@ static int fts5VocabInitVtab(
 ){
  const char *azSchema[] = { 
    "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA  ")", 
-    "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA  ")"
+    "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA  ")",
+    "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")"
  };

  Fts5VocabTable *pRet = 0;
@@ -235,6 +250,15 @@ static int fts5VocabCreateMethod(

 /* 
 ** Implementation of the xBestIndex method.
+**
+** Only constraints of the form:
+**
+**     term <= ?
+**     term == ?
+**     term >= ?
+**
+** are interpreted. Less-than and less-than-or-equal are treated 
+** identically, as are greater-than and greater-than-or-equal.
 */
 static int fts5VocabBestIndexMethod(
  sqlite3_vtab *pUnused,
@@ -279,8 +303,19 @@ static int fts5VocabBestIndexMethod(
    }
  }

-  pInfo->idxNum = idxNum;
+  /* This virtual table always delivers results in ascending order of
+  ** the "term" column (column 0). So if the user has requested this
+  ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the
+  ** sqlite3_index_info.orderByConsumed flag to tell the core the results
+  ** are already in sorted order.  */
+  if( pInfo->nOrderBy==1 
+   && pInfo->aOrderBy[0].iColumn==0 
+   && pInfo->aOrderBy[0].desc==0
+  ){
+    pInfo->orderByConsumed = 1;
+  }

+  pInfo->idxNum = idxNum;
  return SQLITE_OK;
 }

@@ -367,6 +402,54 @@ static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){
  return SQLITE_OK;
 }

+static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){
+  int rc = SQLITE_OK;
+  
+  if( sqlite3Fts5IterEof(pCsr->pIter) ){
+    pCsr->bEof = 1;
+  }else{
+    const char *zTerm;
+    int nTerm;
+    zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
+    if( pCsr->nLeTerm>=0 ){
+      int nCmp = MIN(nTerm, pCsr->nLeTerm);
+      int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
+      if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
+        pCsr->bEof = 1;
+      }
+    }
+
+    sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
+  }
+  return rc;
+}
+
+static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
+  int eDetail = pCsr->pConfig->eDetail;
+  int rc = SQLITE_OK;
+  Fts5IndexIter *pIter = pCsr->pIter;
+  i64 *pp = &pCsr->iInstPos;
+  int *po = &pCsr->iInstOff;
+  
+  while( eDetail==FTS5_DETAIL_NONE
+      || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) 
+  ){
+    pCsr->iInstPos = 0;
+    pCsr->iInstOff = 0;
+
+    rc = sqlite3Fts5IterNextScan(pCsr->pIter);
+    if( rc==SQLITE_OK ){
+      rc = fts5VocabInstanceNewTerm(pCsr);
+      if( eDetail==FTS5_DETAIL_NONE ) break;
+    }
+    if( rc ){
+      pCsr->bEof = 1;
+      break;
+    }
+  }
+
+  return rc;
+}

 /*
 ** Advance the cursor to the next row in the table.
@@ -379,13 +462,17 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){

  pCsr->rowid++;

+  if( pTab->eType==FTS5_VOCAB_INSTANCE ){
+    return fts5VocabInstanceNext(pCsr);
+  }
+
  if( pTab->eType==FTS5_VOCAB_COL ){
    for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
      if( pCsr->aDoc[pCsr->iCol] ) break;
    }
  }

-  if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){
+  if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){
    if( sqlite3Fts5IterEof(pCsr->pIter) ){
      pCsr->bEof = 1;
    }else{
@@ -409,22 +496,26 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){

      assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
      while( rc==SQLITE_OK ){
+        int eDetail = pCsr->pConfig->eDetail;
        const u8 *pPos; int nPos;   /* Position list */
        i64 iPos = 0;               /* 64-bit position read from poslist */
        int iOff = 0;               /* Current offset within position list */

        pPos = pCsr->pIter->pData;
        nPos = pCsr->pIter->nData;
-        switch( pCsr->pConfig->eDetail ){
-          case FTS5_DETAIL_FULL:
-            pPos = pCsr->pIter->pData;
-            nPos = pCsr->pIter->nData;
-            if( pTab->eType==FTS5_VOCAB_ROW ){
+
+        switch( pTab->eType ){
+          case FTS5_VOCAB_ROW:
+            if( eDetail==FTS5_DETAIL_FULL ){
              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
                pCsr->aCnt[0]++;
              }
-              pCsr->aDoc[0]++;
-            }else{
+            }
+            pCsr->aDoc[0]++;
+            break;
+
+          case FTS5_VOCAB_COL:
+            if( eDetail==FTS5_DETAIL_FULL ){
              int iCol = -1;
              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
                int ii = FTS5_POS2COLUMN(iPos);
@@ -438,13 +529,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
                  iCol = ii;
                }
              }
-            }
-            break;
-
-          case FTS5_DETAIL_COLUMNS:
-            if( pTab->eType==FTS5_VOCAB_ROW ){
-              pCsr->aDoc[0]++;
-            }else{
+            }else if( eDetail==FTS5_DETAIL_COLUMNS ){
              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
                assert_nc( iPos>=0 && iPos<nCol );
                if( iPos>=nCol ){
@@ -453,18 +538,21 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
                }
                pCsr->aDoc[iPos]++;
              }
+            }else{
+              assert( eDetail==FTS5_DETAIL_NONE );
+              pCsr->aDoc[0]++;
            }
            break;

-          default: 
-            assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE );
-            pCsr->aDoc[0]++;
+          default:
+            assert( pTab->eType==FTS5_VOCAB_INSTANCE );
            break;
        }

        if( rc==SQLITE_OK ){
          rc = sqlite3Fts5IterNextScan(pCsr->pIter);
        }
+        if( pTab->eType==FTS5_VOCAB_INSTANCE ) break;

        if( rc==SQLITE_OK ){
          zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
@@ -494,7 +582,9 @@ static int fts5VocabFilterMethod(
  int nUnused,                    /* Number of elements in apVal */
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
 ){
+  Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
+  int eType = pTab->eType;
  int rc = SQLITE_OK;

  int iVal = 0;
@@ -534,11 +624,16 @@ static int fts5VocabFilterMethod(
    }
  }

-
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
  }
-  if( rc==SQLITE_OK ){
+  if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){
+    rc = fts5VocabInstanceNewTerm(pCsr);
+  }
+  if( rc==SQLITE_OK 
+   && !pCsr->bEof 
+   && (eType!=FTS5_VOCAB_INSTANCE || pCsr->pConfig->eDetail!=FTS5_DETAIL_NONE)
+  ){
    rc = fts5VocabNextMethod(pCursor);
  }

@@ -580,13 +675,41 @@ static int fts5VocabColumnMethod(
    }else{
      iVal = pCsr->aCnt[pCsr->iCol];
    }
-  }else{
+  }else if( eType==FTS5_VOCAB_ROW ){
    assert( iCol==1 || iCol==2 );
    if( iCol==1 ){
      iVal = pCsr->aDoc[0];
    }else{
      iVal = pCsr->aCnt[0];
    }
+  }else{
+    assert( eType==FTS5_VOCAB_INSTANCE );
+    switch( iCol ){
+      case 1:
+        sqlite3_result_int64(pCtx, pCsr->pIter->iRowid);
+        break;
+      case 2: {
+        int ii = -1;
+        if( eDetail==FTS5_DETAIL_FULL ){
+          ii = FTS5_POS2COLUMN(pCsr->iInstPos);
+        }else if( eDetail==FTS5_DETAIL_COLUMNS ){
+          ii = (int)pCsr->iInstPos;
+        }
+        if( ii>=0 && ii<pCsr->pConfig->nCol ){
+          const char *z = pCsr->pConfig->azCol[ii];
+          sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
+        }
+        break;
+      }
+      default: {
+        assert( iCol==3 );
+        if( eDetail==FTS5_DETAIL_FULL ){
+          int ii = FTS5_POS2OFFSET(pCsr->iInstPos);
+          sqlite3_result_int(pCtx, ii);
+        }
+        break;
+      }
+    }
  }

  if( iVal>0 ) sqlite3_result_int64(pCtx, iVal);
@@ -89,6 +89,29 @@ input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); }
 %destructor expr     { sqlite3Fts5ParseNodeFree($$); }
 %destructor exprlist { sqlite3Fts5ParseNodeFree($$); }

+%type colset {Fts5Colset*}
+%destructor colset { sqlite3_free($$); }
+%type colsetlist {Fts5Colset*}
+%destructor colsetlist { sqlite3_free($$); }
+
+colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
+    A = sqlite3Fts5ParseColsetInvert(pParse, X);
+}
+colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
+colset(A) ::= STRING(X). {
+  A = sqlite3Fts5ParseColset(pParse, 0, &X);
+}
+colset(A) ::= MINUS STRING(X). {
+  A = sqlite3Fts5ParseColset(pParse, 0, &X);
+  A = sqlite3Fts5ParseColsetInvert(pParse, A);
+}
+
+colsetlist(A) ::= colsetlist(Y) STRING(X). { 
+  A = sqlite3Fts5ParseColset(pParse, Y, &X); }
+colsetlist(A) ::= STRING(X). { 
+  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
+}
+
 expr(A) ::= expr(X) AND expr(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
 }
@@ -99,6 +122,10 @@ expr(A) ::= expr(X) NOT expr(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
 }

+expr(A) ::= colset(X) COLON LP expr(Y) RP. {
+  sqlite3Fts5ParseSetColset(pParse, Y, X);
+  A = Y;
+}
 expr(A) ::= LP expr(X) RP. {A = X;}
 expr(A) ::= exprlist(X).   {A = X;}

@@ -111,24 +138,8 @@ cnearset(A) ::= nearset(X). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
 }
 cnearset(A) ::= colset(X) COLON nearset(Y). { 
-  sqlite3Fts5ParseSetColset(pParse, Y, X);
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
-}
-
-%type colset {Fts5Colset*}
-%destructor colset { sqlite3_free($$); }
-%type colsetlist {Fts5Colset*}
-%destructor colsetlist { sqlite3_free($$); }
-
-colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
-colset(A) ::= STRING(X). {
-  A = sqlite3Fts5ParseColset(pParse, 0, &X);
-}
-
-colsetlist(A) ::= colsetlist(Y) STRING(X). { 
-  A = sqlite3Fts5ParseColset(pParse, Y, &X); }
-colsetlist(A) ::= STRING(X). { 
-  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
+  sqlite3Fts5ParseSetColset(pParse, A, X);
 }


@@ -441,7 +441,7 @@ db func funk funk
 do_catchsql_test 16.2 {
  SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
 } {0 {{} -1e-06 {}}}
-# {1 {SQL logic error or missing database}}
+# {1 {SQL logic error}}

 #-------------------------------------------------------------------------
 #
@@ -561,9 +561,37 @@ do_test 20.1 {
  execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
 } $::ids

+#--------------------------------------------------------------------
+# Test that a DROP TABLE may be executed within a transaction that
+# writes to an FTS5 table.
+#
+do_execsql_test 21.0 {
+  CREATE TEMP TABLE t8(a, b);
+  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
+}
+
+do_execsql_test 21.1 {
+  BEGIN;
+    INSERT INTO ft VALUES('a b c');
+    DROP TABLE t8;
+  COMMIT;
+}
+
+do_execsql_test 22.0 {
+  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
+  INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
+  BEGIN;
+    INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
+    DELETE FROM t9 WHERE rowid = 2;
+    INSERT INTO t9(rowid, x) VALUES(3, 'bbb');
+  COMMIT;
+}
+
+do_execsql_test 22.1 {
+  SELECT rowid FROM t9('a*')
+} {1}
+
 }


 finish_test
-
-
@@ -294,4 +294,3 @@ do_execsql_test 7.0 {


 finish_test
-
@@ -276,4 +276,3 @@ foreach {tn expr tclexpr} {
 }

 finish_test
-
@@ -231,7 +231,6 @@ foreach {T create} {
        set res [lsort -integer -increasing $res]
      }
      set n [llength $res]
-      if {$T==5} breakpoint 
      do_execsql_test $T.$bAsc.$tn.$n $sql $res
    }
  }
@@ -242,4 +241,3 @@ foreach {T create} {
 }

 finish_test
-
@@ -309,4 +309,3 @@ foreach {tn q cnt} {
 }

 finish_test
-
@@ -72,42 +72,56 @@ foreach {tn doc res} {
  2.2 {o X o o o o o o} {o [X] o o o o o...}
  2.3 {o o X o o o o o} {o o [X] o o o o...}
  2.4 {o o o X o o o o} {o o o [X] o o o...}
-  2.5 {o o o o X o o o} {...o o o [X] o o o}
-  2.6 {o o o o o X o o} {...o o o o [X] o o}
-  2.7 {o o o o o o X o} {...o o o o o [X] o}
+  2.5 {o o o o X o o o} {o o o o [X] o o...}
+  2.6 {o o o o o X o o} {o o o o o [X] o...}
+  2.7 {o o o o o o X o} {o o o o o o [X]...}
  2.8 {o o o o o o o X} {...o o o o o o [X]}

+  2.9  {o o o o o o o X o}       {...o o o o o [X] o}
+  2.10 {o o o o o o o X o o}     {...o o o o [X] o o}
+  2.11 {o o o o o o o X o o o}   {...o o o [X] o o o}
+  2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...}
+
+
  3.1 {X o o o o o o o o} {[X] o o o o o o...}
  3.2 {o X o o o o o o o} {o [X] o o o o o...}
  3.3 {o o X o o o o o o} {o o [X] o o o o...}
  3.4 {o o o X o o o o o} {o o o [X] o o o...}
-  3.5 {o o o o X o o o o} {...o o o [X] o o o...}
-  3.6 {o o o o o X o o o} {...o o o [X] o o o}
-  3.7 {o o o o o o X o o} {...o o o o [X] o o}
-  3.8 {o o o o o o o X o} {...o o o o o [X] o}
-  3.9 {o o o o o o o o X} {...o o o o o o [X]}
+
+  3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...}
+  3.6 {o o o o o o o o X o o o} {...o o o [X] o o o}
+  3.7 {o o o o o o o o o X o o} {...o o o o [X] o o}
+  3.8 {o o o o o o o o o o X o} {...o o o o o [X] o}
+  3.9 {o o o o o o o o o o o X} {...o o o o o o [X]}

  4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
-  4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
-  4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
+  4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...}
+  4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]}

  5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
-  5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
-  5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
-  5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
+  5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...}
+  5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o}
+  5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]}

  6.1 {X o o o X o o o} {[X] o o o [X] o o...}
  6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
-  6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
-  6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
-  6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
+  6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...}
+  6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o}
+  6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]}

  7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
  7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
-  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
-  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
-  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
-  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
+  7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...}
+  7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o}
+  7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o}
+  7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]}
+
+  8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o}
+      {...o o [X] [X] [X] o o...}
+  8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} 
+      {...o o [X] o o o o...}
+  8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
+      {o o o o [X] o o...}
 } {
  do_snippet_test 1.$tn $doc X $res
 }
@@ -124,25 +138,43 @@ if {[detail_is_full]} {
    2.1 {X Y o o o o o o} {[X Y] o o o o o...}
    2.2 {o X Y o o o o o} {o [X Y] o o o o...}
    2.3 {o o X Y o o o o} {o o [X Y] o o o...}
-    2.4 {o o o X Y o o o} {...o o [X Y] o o o}
-    2.5 {o o o o X Y o o} {...o o o [X Y] o o}
-    2.6 {o o o o o X Y o} {...o o o o [X Y] o}
-    2.7 {o o o o o o X Y} {...o o o o o [X Y]}
+    2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o}
+    2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o}
+    2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o}
+    2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]}

    3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
    3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
    3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
-    3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
-    3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
-    3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
-    3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
-    3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
+    3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...}
+    3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o}
+    3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o}
+    3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o}
+    3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]}
  } {
    do_snippet_test 2.$tn $doc "X + Y" $res
  }
 }

+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+  INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13');
+  SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz');
+} {
+  {1 2 3 4 5...}
+}
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE p1 USING fts5(a, b);
+  INSERT INTO p1 VALUES(
+    'x a a a a a a a a a a',
+    'a a a a a a a a a a a a a a a a a a a x'
+  );
+}
+do_execsql_test 5.1 {
+  SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
+} {{[x] a a a a a...}}
+
 } ;# foreach_detail_mode 

 finish_test
-
@@ -142,4 +142,3 @@ if {[detail_is_full]} {


 finish_test
-
@@ -167,4 +167,3 @@ do_execsql_test 1.8.2 {
 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}

 finish_test
-
@@ -55,4 +55,3 @@ do_execsql_test 1.2 {


 finish_test
-
@@ -66,4 +66,3 @@ do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }


 finish_test
-
@@ -147,4 +147,3 @@ do_execsql_test 3.1 {
 }

 finish_test
-
@@ -77,7 +77,7 @@ foreach {tn defn} {
 } {
  do_test 2.2.$tn {
    catchsql { INSERT INTO ft1(ft1, rank) VALUES('rank', $defn) }
-  } {1 {SQL logic error or missing database}}
+  } {1 {SQL logic error}}
 }

 #-------------------------------------------------------------------------
@@ -297,4 +297,3 @@ do_catchsql_test 4.4.4 {


 finish_test
-
@@ -89,7 +89,6 @@ do_execsql_test 3.1 {
  BEGIN;
    INSERT INTO abc(rowid, a) VALUES(2, 'a');
 }
-breakpoint
 do_execsql_test 3.2 {
    SELECT rowid FROM abc WHERE abc MATCH 'a';
 } {1 2}
@@ -100,4 +99,3 @@ do_execsql_test 3.3 {
 } {1 2}

 finish_test
-
@@ -342,4 +342,3 @@ foreach {tn expr} {


 finish_test
-
@@ -240,7 +240,6 @@ foreach {tn lRow res} {
 } {
  execsql { DELETE FROM x1 }
  foreach row $lRow { execsql { INSERT INTO x1 VALUES($row) } }
-  breakpoint
  do_execsql_test 8.$tn {
    SELECT highlight(x1, 0, '[', ']') FROM x1 WHERE x1 MATCH 'a OR (b AND d)';
  } $res
@@ -279,4 +278,3 @@ do_execsql_test 9.3 {


 finish_test
-
@@ -112,4 +112,3 @@ db eval {
 }

 finish_test
-
@@ -61,4 +61,3 @@ do_test 2.1...slow {
 } {}

 finish_test
-
@@ -64,5 +64,3 @@ foreach_detail_mode $::testprefix {
 }

 finish_test
-
-
@@ -0,0 +1,86 @@
+# 2016 August 10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5colset
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $::testprefix {
+  if {[detail_is_none]} continue
+
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d, detail=%DETAIL%);
+    INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
+    INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
+    INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
+    INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
+  }
+
+  foreach {tn q res} {
+    1 "a"          {1 2 3 4}
+    2 "{a}   : a"  {1}
+    3 "-{a}   : a" {2 3 4}
+    4 "- {a c} : a" {2 4}
+    5 " - {d d c} : a" {1 2}
+    6 "- {d c b a} : a" {}
+    7 "-{\"a\"} : b" {1 2 3}
+    8 "- c : a" {1 2 4}
+    9 "-c : a"  {1 2 4}
+    10 "-\"c\" : a"  {1 2 4}
+  } {
+    do_execsql_test 1.$tn {
+      SELECT rowid FROM t1($q)
+    } $res
+  }
+
+  foreach {tn q res} {
+    0 {{a} : (a AND ":")}     {}
+    1 "{a b c} : (a AND d)"   {2 3}
+    2 "{a b c} : (a AND b:d)" {3}
+    3 "{a b c} : (a AND d:d)" {}
+    4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
+    5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
+    6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
+    7 "{a b c} : (b:a AND c:b)" {2}
+  } {
+    do_execsql_test 2.$tn {
+      SELECT rowid FROM t1($q)
+    } $res
+  }
+
+  foreach {tn w res} {
+    0 "a MATCH 'a'" {1}
+    1 "b MATCH 'a'" {2}
+    2 "b MATCH '{a b c} : a'" {2}
+    3 "b MATCH 'a OR b'"      {1 2}
+    4 "b MATCH 'a OR a:b'"    {2}
+    5 "b MATCH 'a OR b:b'"    {1 2}
+  } {
+    do_execsql_test 3.$tn "
+      SELECT rowid FROM t1 WHERE $w
+    " $res
+  }
+
+  do_catchsql_test 4.1 {
+    SELECT * FROM t1 WHERE rowid MATCH 'a'
+  } {1 {unable to use function MATCH in the requested context}}
+}
+
+
+finish_test
@@ -143,7 +143,6 @@ do_execsql_test 4.1.1 {
  INSERT INTO t5 VALUES('2 4 6 8');
 }

-breakpoint
 do_execsql_test 4.1.2 {
  INSERT INTO t5(t5) VALUES('integrity-check');
 }
@@ -66,7 +66,7 @@ foreach {tn val} {
 } {
  do_catchsql_test 3.$tn {
    INSERT INTO t1(t1, rank) VALUES('rank', $val);
-  } {1 {SQL logic error or missing database}}
+  } {1 {SQL logic error}}
 }

 #-------------------------------------------------------------------------
@@ -110,7 +110,6 @@ do_catchsql_test 5.1 {
  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize="porter 'ascii");
 } {1 {parse error in tokenize directive}} 

-breakpoint
 do_catchsql_test 5.2 {
  CREATE VIRTUAL TABLE xx USING fts5(x, [y[]);
 } {0 {}}
@@ -169,33 +168,33 @@ do_execsql_test 9.0 {
 } {}
 do_catchsql_test 9.1.1 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', -5);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.1.2 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.1.3 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}

 do_catchsql_test 9.2.1 {
  INSERT INTO abc(abc, rank) VALUES('automerge', -5);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.2.2 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 50000000);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.2.3 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 66.67);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_execsql_test 9.2.4 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 1);
 } {}

 do_catchsql_test 9.3.1 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', -5);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.3.2 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_execsql_test 9.3.3 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 1);
 } {}
@@ -205,14 +204,14 @@ do_execsql_test 9.3.4 {

 do_catchsql_test 9.4.1 {
  INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}

 do_catchsql_test 9.5.1 {
  INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.5.2 {
  INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.5.3 {
  INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
 } {0 {}}
@@ -245,7 +244,7 @@ foreach {tn opt} {

 do_catchsql_test 12.1 {
  INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}

 #-------------------------------------------------------------------------
 # errors in the 'usermerge' option
@@ -260,8 +259,7 @@ foreach {tn val} {
  4     1
 } {
  set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)"
-  do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}}
+  do_catchsql_test 13.$tn $sql {1 {SQL logic error}}
 }

 finish_test
-
@@ -66,5 +66,3 @@ do_execsql_test 2.1 {
 }

 finish_test
-
-
@@ -0,0 +1,247 @@
+# 2017 August 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5connect
+
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# The tests in this file test the outcome of a schema-reset happening 
+# within the xConnect() method of an FTS5 table. At one point this
+# was causing a problem in SQLite. Each test proceeds as follows:
+#
+#   1. Connection [db] opens the db and reads from some unrelated, non-FTS5
+#      table causing SQLite to load the db schema into memory.
+#
+#   2. Connection [db2] opens the db and modifies the db schema.
+#
+#   3. Connection [db] reads or writes an existing fts5 table. That the
+#      schema has been modified is detected inside the fts5 xConnect() 
+#      callback that is invoked by sqlite3_prepare(). 
+#
+#   4. Verify that the statement in 3 has worked. SQLite should detect
+#      that the schema has changed and successfully prepare the 
+#      statement against the new schema.
+#
+# Test plan:
+#
+#   1.*: Trigger the xConnect()/schema-reset using statements executed
+#        directly against an FTS5 table.
+#
+#   2.*: Using various statements executed by various BEFORE triggers.
+#
+#   3.*: Using various statements executed by various AFTER triggers.
+#
+#   4.*: Using various statements executed by various INSTEAD OF triggers.
+#
+
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, b);
+  CREATE TABLE abc(x INTEGER PRIMARY KEY);
+  CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b);
+
+  INSERT INTO ft1 VALUES('one', 'two');
+  INSERT INTO ft1 VALUES('three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM ft1" {one two three four}
+  2 "REPLACE INTO ft1(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft1" {five six three four}
+  4 "INSERT INTO ft1 VALUES('seven', 'eight')" {}
+  5 "SELECT * FROM ft1" {five six three four seven eight}
+  6 "DELETE FROM ft1 WHERE rowid=2" {}
+  7 "UPDATE ft1 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft1" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 1.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 1.$tn.2 $sql $res
+
+  do_execsql_test 1.$tn.3 {
+    INSERT INTO ft1(ft1) VALUES('integrity-check');
+  }
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(a, b);
+  CREATE TABLE t2(a, b);
+  CREATE TABLE log(txt);
+
+  CREATE TRIGGER t2_ai AFTER INSERT ON t2 BEGIN
+    INSERT INTO ft2(rowid, a, b) VALUES(new.rowid, new.a, new.b);
+    INSERT INTO log VALUES('insert');
+  END;
+
+  CREATE TRIGGER t2_ad AFTER DELETE ON t2 BEGIN
+    DELETE FROM ft2 WHERE rowid = old.rowid;
+    INSERT INTO log VALUES('delete');
+  END;
+
+  CREATE TRIGGER t2_au AFTER UPDATE ON t2 BEGIN
+    UPDATE ft2 SET a=new.a, b=new.b WHERE rowid=new.rowid;
+    INSERT INTO log VALUES('update');
+  END;
+
+  INSERT INTO t2 VALUES('one', 'two');
+  INSERT INTO t2 VALUES('three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM t2" {one two three four}
+  2 "REPLACE INTO t2(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft2" {five six three four}
+  4 "INSERT INTO t2 VALUES('seven', 'eight')" {}
+  5 "SELECT * FROM ft2" {five six three four seven eight}
+  6 "DELETE FROM t2 WHERE rowid=2" {}
+  7 "UPDATE t2 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft2" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 2.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 2.$tn.2 $sql $res
+
+  do_execsql_test 2.$tn.3 {
+    INSERT INTO ft2(ft2) VALUES('integrity-check');
+  }
+}
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft3 USING fts5(a, b);
+  CREATE TABLE t3(a, b);
+
+  CREATE TRIGGER t3_ai BEFORE INSERT ON t3 BEGIN
+    INSERT INTO ft3(rowid, a, b) VALUES(new.rowid, new.a, new.b);
+    INSERT INTO log VALUES('insert');
+  END;
+
+  CREATE TRIGGER t3_ad BEFORE DELETE ON t3 BEGIN
+    DELETE FROM ft3 WHERE rowid = old.rowid;
+    INSERT INTO log VALUES('delete');
+  END;
+
+  CREATE TRIGGER t3_au BEFORE UPDATE ON t3 BEGIN
+    UPDATE ft3 SET a=new.a, b=new.b WHERE rowid=new.rowid;
+    INSERT INTO log VALUES('update');
+  END;
+
+  INSERT INTO t3(rowid, a, b) VALUES(1, 'one', 'two');
+  INSERT INTO t3(rowid, a, b) VALUES(2, 'three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM t3" {one two three four}
+  2 "REPLACE INTO t3(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft3" {five six three four}
+  4 "INSERT INTO t3(rowid, a, b) VALUES(3, 'seven', 'eight')" {}
+  5 "SELECT * FROM ft3" {five six three four seven eight}
+  6 "DELETE FROM t3 WHERE rowid=2" {}
+  7 "UPDATE t3 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft3" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 3.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 3.$tn.2 $sql $res
+
+  do_execsql_test 3.$tn.3 {
+    INSERT INTO ft3(ft3) VALUES('integrity-check');
+  }
+}
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft4 USING fts5(a, b);
+  CREATE VIEW v4 AS SELECT rowid, * FROM ft4;
+
+  CREATE TRIGGER t4_ai INSTEAD OF INSERT ON v4 BEGIN
+    INSERT INTO ft4(rowid, a, b) VALUES(new.rowid, new.a, new.b);
+    INSERT INTO log VALUES('insert');
+  END;
+
+  CREATE TRIGGER t4_ad INSTEAD OF DELETE ON v4 BEGIN
+    DELETE FROM ft4 WHERE rowid = old.rowid;
+    INSERT INTO log VALUES('delete');
+  END;
+
+  CREATE TRIGGER t4_au INSTEAD OF UPDATE ON v4 BEGIN
+    UPDATE ft4 SET a=new.a, b=new.b WHERE rowid=new.rowid;
+    INSERT INTO log VALUES('update');
+  END;
+
+  INSERT INTO ft4(rowid, a, b) VALUES(1, 'one', 'two');
+  INSERT INTO ft4(rowid, a, b) VALUES(2, 'three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM ft4" {one two three four}
+  2 "REPLACE INTO v4(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft4" {five six three four}
+  4 "INSERT INTO v4(rowid, a, b) VALUES(3, 'seven', 'eight')" {}
+  5 "SELECT * FROM ft4" {five six three four seven eight}
+  6 "DELETE FROM v4 WHERE rowid=2" {}
+  7 "UPDATE v4 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft4" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 4.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 4.$tn.2 $sql $res
+
+  do_execsql_test 4.$tn.3 {
+    INSERT INTO ft3(ft3) VALUES('integrity-check');
+  }
+}
+
+finish_test
+
@@ -255,4 +255,3 @@ do_execsql_test 6.2 {


 finish_test
-
@@ -96,4 +96,3 @@ do_catchsql_test 3.1 {
 } {1 {database disk image is malformed}}

 finish_test
-
@@ -37,7 +37,7 @@ do_execsql_test 1.0 {
 }
 set mask [expr 31 << 31]

-if 1 {
+if 0 {

 # Test 1:
 #
@@ -84,6 +84,8 @@ foreach {tno stmt} {
  }
 }

+}
+
 # Using the same database as the 1.* tests.
 #
 # Run N-1 tests, where N is the number of bytes in the rightmost leaf page
@@ -212,8 +214,6 @@ foreach {tn nCut} {
  # do_test 4.$tn.x { expr $nCorrupt>0 } 1
 }

-}
-
 set doc [string repeat "A B C " 1000]
 do_execsql_test 5.0 {
  CREATE VIRTUAL TABLE x5 USING fts5(tt);
@@ -269,4 +269,3 @@ do_catchsql_test 6.2 {

 sqlite3_fts5_may_be_corrupt 0
 finish_test
-
@@ -409,4 +409,3 @@ do_catchsql_test 9.2.2 {

 sqlite3_fts5_may_be_corrupt 0
 finish_test
-
@@ -0,0 +1,53 @@
+# 2017 May 12
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5delete
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+fts5_aux_test_functions db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<5000
+  )
+  INSERT INTO t1(rowid, x) SELECT i, (i/2)*2 FROM s;
+}
+
+do_test 1.1 {
+  execsql BEGIN
+  for {set i 1} {$i<=5000} {incr i} {
+    if {$i % 2} {
+      execsql { INSERT INTO t1 VALUES($i) }
+    } else {
+      execsql { DELETE FROM t1 WHERE rowid = $i }
+    }
+  }
+  execsql COMMIT
+} {}
+
+do_test 1.2 {
+  execsql { INSERT INTO t1(t1, rank) VALUES('usermerge', 2); }
+  for {set i 0} {$i < 5} {incr i} {
+    execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+  }
+} {}
+
+finish_test
@@ -241,4 +241,3 @@ do_execsql_test 5.3 {


 finish_test
-
@@ -63,5 +63,3 @@ foreach_detail_mode $::testprefix {


 finish_test
-
-
@@ -66,7 +66,6 @@ proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} {
  }
  execsql COMMIT

-  breakpoint
  do_test $tn.1 {
    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
  } {}
@@ -124,7 +123,6 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} {
  do_execsql_test $tn.1 {
    SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a'
  } {1}
-  breakpoint
  do_execsql_test $tn.2 {
    SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
  } {1}
@@ -197,4 +195,3 @@ foreach v $vocab {


 finish_test
-
@@ -44,4 +44,3 @@ do_execsql_test 1.2 {


 finish_test
-
@@ -63,7 +63,21 @@ do_catchsql_test 2.1 {
  SELECT fts5_expr_tcl()
 } {1 {wrong number of arguments to function fts5_expr_tcl}}

+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE e1 USING fts5(text, tokenize = 'porter unicode61');
+  INSERT INTO e1 VALUES ("just a few words with a / inside");
+}
+do_execsql_test 3.1 {
+  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
+} {1 -1e-06}
+do_execsql_test 3.2 {
+  SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"'
+} 1
+do_execsql_test 3.3 {
+  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
+} {1 -1e-06}
+
+
+
 finish_test
-
-
-
@@ -351,4 +351,3 @@ do_faultsim_test 9.1 -faults oom-* -prep {


 finish_test
-
@@ -137,4 +137,3 @@ do_faultsim_test 5.0 -faults oom-* -prep {
 }

 finish_test
-
@@ -110,4 +110,3 @@ do_faultsim_test 3.2 -faults oom-* -prep {


 finish_test
-
@@ -395,4 +395,3 @@ do_faultsim_test 14.1 -faults oom-t* -prep {
 }

 finish_test
-
@@ -105,7 +105,6 @@ do_faultsim_test 3.2 -faults oom-t* -body {
  faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
 }

-breakpoint
 do_execsql_test 3.3.0 {
  SELECT * FROM tv2;
 } {
@@ -130,4 +129,3 @@ do_faultsim_test 3.3 -faults oom-t* -body {


 finish_test
-
@@ -280,7 +280,6 @@ do_faultsim_test 5.4 -faults oom* -prep {

 #-------------------------------------------------------------------------
 catch { db close }
-breakpoint
 do_faultsim_test 6 -faults oom* -prep {
  sqlite_orig db test.db
  sqlite3_db_config_lookaside db 0 0 0
@@ -292,4 +291,3 @@ do_faultsim_test 6 -faults oom* -prep {
  db close
 }
 finish_test
-
@@ -116,4 +116,3 @@ do_faultsim_test 2.2 -faults oom-* -body {
 }

 finish_test
-
@@ -82,4 +82,3 @@ do_faultsim_test 4 -faults oom-* -prep {


 finish_test
-
@@ -153,4 +153,3 @@ do_faultsim_test 6 -faults oom-* -body {
 } ;# foreach_detail_mode...

 finish_test
-
@@ -61,4 +61,3 @@ do_faultsim_test 2 -faults oom* -prep {
  faultsim_test_result {0 {1 2}}
 }
 finish_test
-
@@ -78,6 +78,57 @@ do_faultsim_test 2.4 -faults oom* -body {
  faultsim_test_result {0 {{3 2} {2 3}}} 
 }

+#-------------------------------------------------------------------------
+#
+reset_db 
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(z);
+}
+
+do_faultsim_test 3.1 -faults oom* -body {
+  execsql {
+    SELECT rowid FROM x1('c') WHERE rowid>1;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_execsql_test 3.2 {
+  INSERT INTO x1 VALUES('a b c');
+  INSERT INTO x1 VALUES('b c d');
+  INSERT INTO x1 VALUES('c d e');
+  INSERT INTO x1 VALUES('d e f');
+}
+do_faultsim_test 3.3 -faults oom* -body {
+  execsql {
+    SELECT rowid FROM x1('c') WHERE rowid>1;
+  }
+} -test {
+  faultsim_test_result {0 {2 3}}
+}
+
+#-------------------------------------------------------------------------
+# Test OOM injection with nested colsets.
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d);
+  INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
+  INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
+  INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
+  INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
+}
+do_faultsim_test 4.1 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); }
+} -test {
+  faultsim_test_result {0 2}
+}
+
+do_faultsim_test 4.2 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
+} -test {
+  faultsim_test_result {0 {2 3}}
+}
+

 finish_test
-
@@ -0,0 +1,87 @@
+# 2016 February 2
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultA
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+  if {"%DETAIL%"=="none"} continue
+
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE o1 USING fts5(a, b, c, detail=%DETAIL%);
+    INSERT INTO o1(o1, rank) VALUES('pgsz', 32);
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
+    INSERT INTO o1 SELECT 'A', 'B', 'C' FROM s;
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
+    INSERT INTO o1 SELECT 'C', 'A', 'B' FROM s;
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
+    INSERT INTO o1 SELECT 'B', 'C', 'A' FROM s;
+  }
+  
+  do_faultsim_test 1 -faults int* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT count(*) FROM o1('a') }
+  } -test {
+    faultsim_test_result {0 180} {1 {vtable constructor failed: o1}}
+  }
+
+  do_faultsim_test 2 -faults int* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT * FROM o1('a:a AND {b c}:b') ORDER BY rank }
+    expr 1
+  } -test {
+    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
+  }
+
+  do_faultsim_test 3 -faults int* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT * FROM o1('{b c}:b NOT a:a') ORDER BY rank }
+    expr 1
+  } -test {
+    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
+  }
+
+  do_faultsim_test 4 -faults int* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT * FROM o1('b:b OR a:a') }
+    expr 1
+  } -test {
+    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
+  }
+
+  do_faultsim_test 5 -faults int* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT count(*) FROM o1('c:b') }
+    expr 1
+  } -test {
+    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
+  }
+}
+
+finish_test
@@ -40,4 +40,3 @@ do_test 1.1 {


 finish_test
-
@@ -90,4 +90,3 @@ do_catchsql_test 4.1 {
 } {1 {fts5: syntax error near "`"}}

 finish_test
-
@@ -121,7 +121,6 @@ foreach_detail_mode $testprefix {
    }

    execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) }
-breakpoint
    execsql {
      INSERT INTO t2 VALUES($small || ' ' || $big);
    }
@@ -130,4 +129,3 @@ breakpoint
 } ;# foreach_detail_mode

 finish_test
-
@@ -210,4 +210,3 @@ foreach {tn pgsz} {
 }

 finish_test
-
@@ -0,0 +1,72 @@
+# 2017 Feb 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests of the last_insert_rowid functionality with fts5.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5lastrowid
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(str);
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO t1 VALUES('one string');
+  INSERT INTO t1 VALUES('two string');
+  INSERT INTO t1 VALUES('three string');
+  SELECT last_insert_rowid();
+} {3}
+
+do_execsql_test 1.2 {
+  BEGIN;
+    INSERT INTO t1 VALUES('one string');
+    INSERT INTO t1 VALUES('two string');
+    INSERT INTO t1 VALUES('three string');
+  COMMIT;
+  SELECT last_insert_rowid();
+} {6}
+
+do_execsql_test 1.3 {
+  INSERT INTO t1(rowid, str) VALUES(-22, 'some more text');
+  SELECT last_insert_rowid();
+} {-22}
+
+do_execsql_test 1.4 {
+  BEGIN;
+    INSERT INTO t1(rowid, str) VALUES(45, 'some more text');
+    INSERT INTO t1(rowid, str) VALUES(46, 'some more text');
+    INSERT INTO t1(rowid, str) VALUES(222, 'some more text');
+    SELECT last_insert_rowid();
+  COMMIT;
+  SELECT last_insert_rowid();
+} {222 222}
+
+do_execsql_test 1.5 {
+  CREATE TABLE x1(x);
+  INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo');
+  INSERT INTO t1 SELECT x FROM x1;
+  SELECT last_insert_rowid();
+} {226}
+
+do_execsql_test 1.6 {
+  INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1;
+  SELECT last_insert_rowid();
+} {14}
+
+
+finish_test
@@ -0,0 +1,43 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5leftjoin
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE vt USING fts5(x);
+  INSERT INTO vt VALUES('abc');
+  INSERT INTO vt VALUES('xyz');
+
+  CREATE TABLE t1(a INTEGER PRIMARY KEY);
+  INSERT INTO t1 VALUES(1), (2);
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM t1 LEFT JOIN (
+    SELECT rowid AS rrr, * FROM vt WHERE vt MATCH 'abc'
+  ) ON t1.a = rrr
+} {1 1 abc 2 {} {}}
+
+do_execsql_test 1.2 {
+  SELECT * FROM t1 LEFT JOIN vt ON (vt MATCH 'abc')
+} {1 abc 2 abc}
+
+finish_test
@@ -472,7 +472,7 @@ do_execsql_test 12.1 {
 #
 reset_db
 proc xyz {} {}
-db func fts5 -argcount 0 xyz
+db func fts5 -argcount 1 xyz
 do_test 13.1 {
  list [catch { sqlite3_fts5_register_matchinfo db } msg] $msg
 } {1 SQLITE_ERROR}
@@ -492,4 +492,3 @@ do_catchsql_test 14.2 {
 } {1 {unrecognized matchinfo flag: d}}

 finish_test
-
@@ -241,4 +241,3 @@ do_execsql_test 6.3 {


 finish_test
-
@@ -55,4 +55,3 @@ do_execsql_test 1.2 {
 }

 finish_test
-
@@ -45,4 +45,3 @@ do_multiclient_test tn {
 };# do_multiclient_test
 };# foreach_detail_mode
 finish_test
-
@@ -68,4 +68,3 @@ do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0


 finish_test
-
@@ -178,4 +178,3 @@ do_execsql_test 4.3.1 {
 do_test 4.2.2 { fts5_level_segs ttt } {3}

 finish_test
-
@@ -106,4 +106,3 @@ foreach {tn nStep} {
  do_test 2.$tn.6 { fts5_segcount t1 } 1
 }
 finish_test
-
@@ -116,4 +116,3 @@ do_execsql_test 2.0 {
 }

 finish_test
-
@@ -30,7 +30,7 @@ do_eqp_test 1.1 {
  SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
 } {
  0 0 0 {SCAN TABLE t1} 
-  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
+  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
 }

 do_eqp_test 1.2 {
@@ -43,7 +43,7 @@ do_eqp_test 1.2 {
 do_eqp_test 1.3 {
  SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
 } {
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
+  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
 }

@@ -64,4 +64,3 @@ do_eqp_test 1.5 {


 finish_test
-
@@ -11803,4 +11803,3 @@ foreach {in out} $test_vocab {


 finish_test
-
@@ -67,4 +67,3 @@ foreach {in out} $test_vocab {


 finish_test
-
@@ -9,7 +9,7 @@
 #
 #***********************************************************************
 #
-# This file containst tests focused on prefix indexes.
+# This file contains tests focused on prefix indexes.
 #

 source [file join [file dirname [info script]] fts5_common.tcl]
@@ -341,5 +341,3 @@ foreach {tn create} {
 }

 finish_test
-
-
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .14.0
 .22.0
				`@@ -66,4 +66,3 @@ do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }`


				`finish_test`
				`@@ -63,5 +63,3 @@ foreach_detail_mode $::testprefix {`


				`finish_test`
				`@@ -351,4 +351,3 @@ do_faultsim_test 9.1 -faults oom-* -prep {`


				`finish_test`
				`@@ -110,4 +110,3 @@ do_faultsim_test 3.2 -faults oom-* -prep {`


				`finish_test`
				`@@ -82,4 +82,3 @@ do_faultsim_test 4 -faults oom-* -prep {`


				`finish_test`
				`@@ -68,4 +68,3 @@ do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0`


				`finish_test`
				`@@ -11803,4 +11803,3 @@ foreach {in out} $test_vocab {`


				`finish_test`
				`@@ -67,4 +67,3 @@ foreach {in out} $test_vocab {`


				`finish_test`