Fix an incorrect type signature for the loadable extension pointer

for sqlite3_result_pointer(). FossilOrigin-Name: 0bd7875bd9948836a14061275eb8ddac627f562a49f59f400ec98c00e2be82c5
Add the three new pointer interfaces to the loadable extension mechanism.
2017-07-13 17:56:52 +00:00 · 2017-07-13 17:54:12 +00:00 · 2017-07-13 17:48:14 +00:00 · 2017-07-13 17:34:46 +00:00 · 2017-07-12 20:43:23 +00:00 · 2017-07-12 18:05:54 +00:00
732 changed files with 111849 additions and 14407 deletions
@@ -30,7 +30,8 @@ BCC = @BUILD_CC@ @BUILD_CFLAGS@
 #
 CC = @CC@
 CFLAGS = @CPPFLAGS@ @CFLAGS@
-TCC = $(CC) $(CFLAGS) -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/fts3
+TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
+TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session

 # Define this for the autoconf-based build, so that the code knows it can
 # include the generated config.h
@@ -173,13 +174,14 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
         fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
 	 fts5.lo \
         func.lo global.lo hash.lo \
-         icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \
+         icu.lo insert.lo json1.lo legacy.lo loadext.lo \
         main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
         memjournal.lo \
         mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
         notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
         pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
-         random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
+         random.lo resolve.lo rowset.lo rtree.lo \
+         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
         table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
         update.lo util.lo vacuum.lo \
         vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
@@ -224,7 +226,6 @@ SRC = \
  $(TOP)/src/hash.h \
  $(TOP)/src/hwtime.h \
  $(TOP)/src/insert.c \
-  $(TOP)/src/journal.c \
  $(TOP)/src/legacy.c \
  $(TOP)/src/loadext.c \
  $(TOP)/src/main.c \
@@ -342,13 +343,15 @@ SRC += \
 SRC += \
  $(TOP)/ext/rtree/rtree.h \
  $(TOP)/ext/rtree/rtree.c
+SRC += \
+  $(TOP)/ext/session/sqlite3session.c \
+  $(TOP)/ext/session/sqlite3session.h
 SRC += \
  $(TOP)/ext/rbu/sqlite3rbu.h \
  $(TOP)/ext/rbu/sqlite3rbu.c
 SRC += \
-  $(TOP)/ext/misc/json1.c
-
-
+  $(TOP)/ext/misc/json1.c \
+  $(TOP)/ext/misc/stmt.c

 # Generated source code files
 #
@@ -376,9 +379,11 @@ TESTSRC = \
  $(TOP)/src/test_autoext.c \
  $(TOP)/src/test_async.c \
  $(TOP)/src/test_backup.c \
+  $(TOP)/src/test_bestindex.c \
  $(TOP)/src/test_blob.c \
  $(TOP)/src/test_btree.c \
  $(TOP)/src/test_config.c \
+  $(TOP)/src/test_delete.c \
  $(TOP)/src/test_demovfs.c \
  $(TOP)/src/test_devsym.c \
  $(TOP)/src/test_fs.c \
@@ -405,14 +410,17 @@ TESTSRC = \
  $(TOP)/src/test_windirent.c \
  $(TOP)/src/test_wsd.c       \
  $(TOP)/ext/fts3/fts3_term.c \
-  $(TOP)/ext/fts3/fts3_test.c \
+  $(TOP)/ext/fts3/fts3_test.c  \
+  $(TOP)/ext/session/test_session.c \
  $(TOP)/ext/rbu/test_rbu.c 

 # Statically linked extensions
 #
 TESTSRC += \
  $(TOP)/ext/misc/amatch.c \
+  $(TOP)/ext/misc/carray.c \
  $(TOP)/ext/misc/closure.c \
+  $(TOP)/ext/misc/csv.c \
  $(TOP)/ext/misc/eval.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
@@ -423,6 +431,7 @@ TESTSRC += \
  $(TOP)/ext/misc/nextchar.c \
  $(TOP)/ext/misc/percentile.c \
  $(TOP)/ext/misc/regexp.c \
+  $(TOP)/ext/misc/remember.c \
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \
@@ -474,7 +483,8 @@ TESTSRC2 = \
  $(TOP)/ext/fts3/fts3_term.c \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_write.c \
-  $(TOP)/ext/async/sqlite3async.c
+  $(TOP)/ext/async/sqlite3async.c \
+  $(TOP)/ext/session/sqlite3session.c 

 # Header files used by all library source files.
 #
@@ -533,7 +543,8 @@ TESTPROGS = \
  testfixture$(TEXE) \
  sqlite3$(TEXE) \
  sqlite3_analyzer$(TEXE) \
-  sqldiff$(TEXE)
+  sqldiff$(TEXE) \
+  dbhash$(TEXE)

 # Databases containing fuzzer test cases
 #
@@ -541,7 +552,8 @@ FUZZDATA = \
  $(TOP)/test/fuzzdata1.db \
  $(TOP)/test/fuzzdata2.db \
  $(TOP)/test/fuzzdata3.db \
-  $(TOP)/test/fuzzdata4.db
+  $(TOP)/test/fuzzdata4.db \
+  $(TOP)/test/fuzzdata5.db

 # Standard options to testfixture
 #
@@ -550,9 +562,15 @@ TESTOPTS = --verbose=file --output=test-out.txt
 # Extra compiler options for various shell tools
 #
 SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
+# SHELL_OPT += -DSQLITE_ENABLE_FTS5
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
+SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
+SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
+FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
+FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
+DBFUZZ_OPT = 

 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
@@ -581,8 +599,15 @@ sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
 		$(TOP)/src/shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"

-sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
-	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
+sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
+	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)
+
+dbhash$(TEXE):	$(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h
+	$(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS)
+
+scrub$(TEXE):	$(TOP)/ext/misc/scrub.c sqlite3.lo
+	$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
+		$(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS)

 srcck1$(BEXE):	$(TOP)/tool/srcck1.c
 	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
@@ -594,11 +619,18 @@ fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
 	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)

-fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
-	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
+fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h
+	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)

-mptester$(TEXE):	sqlite3.c $(TOP)/mptest/mptest.c
-	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
+ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
+	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
+             $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
+
+dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
+	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
+
+mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
+	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
 		$(TLIBS) -rpath "$(libdir)"

 MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
@@ -634,6 +666,7 @@ mptest:	mptester$(TEXE)
 sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
 	cp tsrc/shell.c tsrc/sqlite3ext.h .
+	cp $(TOP)/ext/session/sqlite3session.h .

 sqlite3ext.h:	.target_source
 	cp tsrc/sqlite3ext.h .
@@ -738,9 +771,6 @@ hash.lo:	$(TOP)/src/hash.c $(HDR)
 insert.lo:	$(TOP)/src/insert.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c

-journal.lo:	$(TOP)/src/journal.c $(HDR)
-	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/journal.c
-
 legacy.lo:	$(TOP)/src/legacy.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c

@@ -1000,9 +1030,15 @@ fts3_write.lo:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
 rtree.lo:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c

+sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
+
 json1.lo:	$(TOP)/ext/misc/json1.c
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c

+stmt.lo:	$(TOP)/ext/misc/stmt.c
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c
+
 # FTS5 things
 #
 FTS5_SRC = \
@@ -1025,7 +1061,7 @@ FTS5_SRC = \
 fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
 	cp $(TOP)/ext/fts5/fts5parse.y .
 	rm -f fts5parse.h
-	./lemon $(OPTS) fts5parse.y
+	./lemon$(BEXE) $(OPTS) fts5parse.y

 fts5parse.h: fts5parse.c

@@ -1050,6 +1086,9 @@ sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
 TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
+TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
+TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
+TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB

 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
 TESTFIXTURE_SRC1 = sqlite3.c
@@ -1082,6 +1121,11 @@ fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
 valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
 	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)

+# The veryquick.test TCL tests.
+#
+tcltest:	./testfixture$(TEXE)
+	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
+
 # Minimal testing that runs in less than 3 minutes
 #
 quicktest:	./testfixture$(TEXE)
@@ -1090,8 +1134,7 @@ quicktest:	./testfixture$(TEXE)
 # This is the common case.  Run many tests that do not take too long,
 # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
 #
-test:	$(TESTPROGS) sourcetest fastfuzztest
-	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
+test:	fastfuzztest sourcetest $(TESTPROGS) tcltest

 # Run a test using valgrind.  This can take a really long time
 # because valgrind is so much slower than a native machine.
@@ -1118,6 +1161,10 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)

+dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
+	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
+           $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
+
 showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)

@@ -1130,17 +1177,25 @@ showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
 showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)

+changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
+
 rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)

 LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c

-wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.c
-	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS)
+wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)

-speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
-	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
+speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.c
+	$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)
+
+KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
+
+kvtest$(TEXE):	$(TOP)/test/kvtest.c sqlite3.c
+	$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)

 rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
 	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
@@ -1152,8 +1207,9 @@ loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
 # symbols that do not begin with "sqlite3_". It is run as part of the
 # releasetest.tcl script.
 #
+VALIDIDS=' sqlite3(changeset|changegroup|session)?_'
 checksymbols: sqlite3.lo
-	nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0
+	nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0
 	echo '0 errors out of 1 tests'

 # Build the amalgamation-autoconf package.  The amalamgation-tarball target builds
@@ -1216,13 +1272,12 @@ clean:
 	rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz
 	rm -f mkkeywordhash$(BEXE) keywordhash.h
 	rm -f *.da *.bb *.bbg gmon.out
-	rm -rf quota2a quota2b quota2c
 	rm -rf tsrc .target_source
 	rm -f tclsqlite3$(TEXE)
 	rm -f testfixture$(TEXE) test.db
 	rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE)
 	rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE)
-	rm -f wordcount$(TEXE)
+	rm -f wordcount$(TEXE) changeset$(TEXE)
 	rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
 	rm -f sqlite3.c
 	rm -f sqlite3rc.h
@@ -1235,6 +1290,7 @@ clean:
 	rm -f fuzzershell fuzzershell.exe
 	rm -f fuzzcheck fuzzcheck.exe
 	rm -f sqldiff sqldiff.exe
+	rm -f dbhash dbhash.exe
 	rm -f fts5.* fts5parse.*

 distclean:	clean
@@ -8,6 +8,50 @@ If you are reading this on a Git mirror someplace, you are doing it wrong.
 The [official repository](https://www.sqlite.org/src/) is better.  Go there
 now.

+## Obtaining The Code
+
+SQLite sources are managed using the
+[Fossil](https://www.fossil-scm.org/), a distributed version control system
+that was specifically designed to support SQLite development.
+If you do not want to use Fossil, you can download tarballs or ZIP
+archives as follows:
+
+  *  Lastest trunk check-in:
+     <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
+     <https://www.sqlite.org/src/zip/sqlite.zip>.
+
+  *  Latest release:
+     <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
+     <https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
+
+  *  For other check-ins, substitute an appropriate branch name or
+     tag or hash prefix for "release" in the URLs of the previous
+     bullet.  Or browse the [timeline](https://www.sqlite.org/src/timeline)
+     to locate the check-in desired, click on its information page link,
+     then click on the "Tarball" or "ZIP Archive" links on the information
+     page.
+
+If you do want to use Fossil to check out the source tree, 
+first install Fossil version 2.0 or later.
+(Source tarballs and precompiled binaries available
+[here](https://www.fossil-scm.org/fossil/uv/download.html).  Fossil is
+a stand-alone program.  To install, simply download or build the single 
+executable file and put that file someplace on your $PATH.)
+Then run commands like this:
+
+        mkdir ~/sqlite
+        cd ~/sqlite
+        fossil clone https://www.sqlite.org/src sqlite.fossil
+        fossil open sqlite.fossil
+    
+After setting up a repository using the steps above, you can always
+update to the lastest version using:
+
+        fossil update trunk   ;# latest trunk check-in
+        fossil update release ;# latest official release
+
+Or type "fossil ui" to get a web-based user interface.
+
 ## Compiling

 First create a directory in which to place
@@ -18,13 +62,13 @@ script found at the root of the source tree.  Then run "make".

 For example:

-    tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
-    mkdir bld                ;#  Build will occur in a sibling directory
-    cd bld                   ;#  Change to the build directory
-    ../sqlite/configure      ;#  Run the configure script
-    make                     ;#  Run the makefile.
-    make sqlite3.c           ;#  Build the "amalgamation" source file
-    make test                ;#  Run some tests (requires Tcl)
+        tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
+        mkdir bld                ;#  Build will occur in a sibling directory
+        cd bld                   ;#  Change to the build directory
+        ../sqlite/configure      ;#  Run the configure script
+        make                     ;#  Run the makefile.
+        make sqlite3.c           ;#  Build the "amalgamation" source file
+        make test                ;#  Run some tests (requires Tcl)

 See the makefile for additional targets.

@@ -43,13 +87,13 @@ with the provided "Makefile.msc" to build one of the supported targets.

 For example:

-    mkdir bld
-    cd bld
-    nmake /f Makefile.msc TOP=..\sqlite
-    nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
-    nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
-    nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
-    nmake /f Makefile.msc test TOP=..\sqlite
+        mkdir bld
+        cd bld
+        nmake /f Makefile.msc TOP=..\sqlite
+        nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
+        nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
+        nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
+        nmake /f Makefile.msc test TOP=..\sqlite

 There are several build options that can be set via the NMAKE command
 line.  For example, to build for WinRT, simply add "FOR_WINRT=1" argument
@@ -64,19 +108,22 @@ The makefiles also require AWK.

 ## Source Code Tour

-Most of the core source files are in the **src/** subdirectory.  But
-src/ also contains files used to build the "testfixture" test harness;
-those file all begin with "test".  And src/ contains the "shell.c" file
-which is the main program for the "sqlite3.exe" command-line shell and
-the "tclsqlite.c" file which implements the bindings to SQLite from the
-Tcl programming language.  (Historical note:  SQLite began as a Tcl
+Most of the core source files are in the **src/** subdirectory.  The
+**src/** folder also contains files used to build the "testfixture" test
+harness. The names of the source files used by "testfixture" all begin
+with "test".
+The **src/** also contains the "shell.c" file
+which is the main program for the "sqlite3.exe"
+[command-line shell](https://sqlite.org/cli.html) and
+the "tclsqlite.c" file which implements the
+[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite.
+(Historical note:  SQLite began as a Tcl
 extension and only later escaped to the wild as an independent library.)

 Test scripts and programs are found in the **test/** subdirectory.
-There are other test suites for SQLite (see
-[How SQLite Is Tested](http://www.sqlite.org/testing.html))
-but those other test suites are
-in separate source repositories.
+Addtional test code is found in other source repositories.
+See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
+additional information.

 The **ext/** subdirectory contains code for extensions.  The
 Full-text search engine is in **ext/fts3**.  The R-Tree engine is in
@@ -100,7 +147,7 @@ manually-edited files and automatically-generated files.
 The SQLite interface is defined by the **sqlite3.h** header file, which is
 generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
 [Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
-The manifest.uuid file contains the SHA1 hash of the particular check-in
+The manifest.uuid file contains the SHA3 hash of the particular check-in
 and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
 contains the current SQLite version number.  The sqlite3.h header is really
 just a copy of src/sqlite.h.in with the source-id and version number inserted
@@ -111,9 +158,8 @@ used to generate that documentation are in a separate source repository.
 The SQL language parser is **parse.c** which is generate from a grammar in
 the src/parse.y file.  The conversion of "parse.y" into "parse.c" is done
 by the [lemon](./doc/lemon.html) LALR(1) parser generator.  The source code
-for lemon is at tool/lemon.c.  Lemon uses a
-template for generating its parser.  A generic template is in tool/lempar.c,
-but SQLite uses a slightly modified template found in src/lempar.c.
+for lemon is at tool/lemon.c.  Lemon uses the tool/lempar.c file as a
+template for generating its parser.

 Lemon also generates the **parse.h** header file, at the same time it
 generates parse.c. But the parse.h header file is
@@ -133,6 +179,13 @@ that maps SQL language keywords (ex: "CREATE", "SELECT", "INDEX", etc.) into
 the numeric codes used by the parse.c parser.  The keywordhash.h file is
 generated by a C-language program at tool mkkeywordhash.c.

+The **pragma.h** header file contains various definitions used to parse
+and implement the PRAGMA statements.  The header is generated by a
+script **tool/mkpragmatab.tcl**. If you want to add a new PRAGMA, edit
+the **tool/mkpragmatab.tcl** file to insert the information needed by the
+parser for your new PRAGMA, then run the script to regenerate the
+**pragma.h** header file.
+
 ### The Amalgamation

 All of the individual C source code and header files (both manually-edited
@@ -150,7 +203,7 @@ subdirectory (using the equivalent of "make target_source") then the
 tool/mksqlite3c.tcl script is run to copy them all together in just the
 right order while resolving internal "#include" references.

-The amalgamation source file is more than 100K lines long.  Some symbolic
+The amalgamation source file is more than 200K lines long.  Some symbolic
 debuggers (most notably MSVC) are unable to deal with files longer than 64K
 lines.  To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
 can be run on the amalgamation to break it up into a single small C file
@@ -167,14 +220,15 @@ See the [architectural description](http://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
 (helping to understand how SQLite works include the
 [file format](http://www.sqlite.org/fileformat2.html) description,
-the [virtual machine](http://www.sqlite.org/vdbe.html) that runs
+the [virtual machine](http://www.sqlite.org/opcode.html) that runs
 prepared statements, the description of
 [how transactions work](http://www.sqlite.org/atomiccommit.html), and
 the [overview of the query planner](http://www.sqlite.org/optoverview.html).

-Unfortunately, years of effort have gone into optimizating SQLite, both
+Years of effort have gone into optimizating SQLite, both
 for small size and high performance.  And optimizations tend to result in
-complex code.  So there is a lot of complexity in the SQLite implementation.
+complex code.  So there is a lot of complexity in the current SQLite
+implementation.  It will not be the easiest library in the world to hack.

 Key files:

@@ -185,7 +239,7 @@ Key files:
  *  **sqliteInt.h** - this header file defines many of the data objects
     used internally by SQLite.

-  *  **parse.y** - This file describes the LALR(1) grammer that SQLite uses
+  *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
     to parse SQL statements, and the actions that are taken at each step
     in the parsing process.

@@ -218,13 +272,13 @@ Key files:
     is not part of the core SQLite library.  But as most of the tests in this
     repository are written in Tcl, the Tcl language bindings are important.

-There are many other source files.  Each has a suscinct header comment that
+There are many other source files.  Each has a succinct header comment that
 describes its purpose and role within the larger system.


 ## Contacts

 The main SQLite webpage is [http://www.sqlite.org/](http://www.sqlite.org/)
-with geographically distributed backup servers at
+with geographically distributed backups at
 [http://www2.sqlite.org/](http://www2.sqlite.org) and
 [http://www3.sqlite.org/](http://www3.sqlite.org).
@@ -1 +1 @@
-3.11.0
+3.20.0
@@ -1,19 +1,20 @@

-AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
+AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE

 lib_LTLIBRARIES = libsqlite3.la
 libsqlite3_la_SOURCES = sqlite3.c
 libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8

 bin_PROGRAMS = sqlite3
-sqlite3_SOURCES = shell.c sqlite3.c sqlite3.h
-sqlite3_LDADD = @READLINE_LIBS@
+sqlite3_SOURCES = shell.c sqlite3.h
+EXTRA_sqlite3_SOURCES = sqlite3.c
+sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
 sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
 sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS

 include_HEADERS = sqlite3.h sqlite3ext.h

-EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt
+EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
 pkgconfigdir = ${libdir}/pkgconfig
 pkgconfig_DATA = sqlite3.pc

@@ -21,7 +21,28 @@ TOP = .
 # Set this non-0 to enable full warnings (-W4, etc) when compiling.
 #
 !IFNDEF USE_FULLWARN
-USE_FULLWARN = 0
+USE_FULLWARN = 1
+!ENDIF
+
+# Set this non-0 to enable treating warnings as errors (-WX, etc) when
+# compiling.
+#
+!IFNDEF USE_FATAL_WARN
+USE_FATAL_WARN = 0
+!ENDIF
+
+# Set this non-0 to enable full runtime error checks (-RTC1, etc).  This
+# has no effect if (any) optimizations are enabled.
+#
+!IFNDEF USE_RUNTIME_CHECKS
+USE_RUNTIME_CHECKS = 0
+!ENDIF
+
+# Set this non-0 to create a SQLite amalgamation file that excludes the
+# various built-in extensions.
+#
+!IFNDEF MINIMAL_AMALGAMATION
+MINIMAL_AMALGAMATION = 0
 !ENDIF

 # Set this non-0 to use "stdcall" calling convention for the core library
@@ -119,12 +140,12 @@ USE_RC = 1
 FOR_WINRT = 0
 !ENDIF

-# Set this non-0 to compile binaries suitable for the UAP environment.
+# Set this non-0 to compile binaries suitable for the UWP environment.
 # This setting does not apply to any binaries that require Tcl to operate
 # properly (i.e. the text fixture, etc).
 #
-!IFNDEF FOR_UAP
-FOR_UAP = 0
+!IFNDEF FOR_UWP
+FOR_UWP = 0
 !ENDIF

 # Set this non-0 to compile binaries suitable for the Windows 10 platform.
@@ -183,6 +204,12 @@ DEBUG = 0
 OPTIMIZATIONS = 2
 !ENDIF

+# Set this to non-0 to enable support for the session extension.
+#
+!IFNDEF SESSION
+SESSION = 0
+!ENDIF
+
 # Set the source code file to be used by executables and libraries when
 # they need the amalgamation.
 #
@@ -204,37 +231,64 @@ SQLITE3H = sqlite3.h
 # This is the name to use for the SQLite dynamic link library (DLL).
 #
 !IFNDEF SQLITE3DLL
+!IF $(FOR_WIN10)!=0
+SQLITE3DLL = winsqlite3.dll
+!ELSE
 SQLITE3DLL = sqlite3.dll
 !ENDIF
+!ENDIF

 # This is the name to use for the SQLite import library (LIB).
 #
 !IFNDEF SQLITE3LIB
+!IF $(FOR_WIN10)!=0
+SQLITE3LIB = winsqlite3.lib
+!ELSE
 SQLITE3LIB = sqlite3.lib
 !ENDIF
+!ENDIF

 # This is the name to use for the SQLite shell executable (EXE).
 #
 !IFNDEF SQLITE3EXE
+!IF $(FOR_WIN10)!=0
+SQLITE3EXE = winsqlite3shell.exe
+!ELSE
 SQLITE3EXE = sqlite3.exe
 !ENDIF
+!ENDIF

 # This is the argument used to set the program database (PDB) file for the
 # SQLite shell executable (EXE).
 #
 !IFNDEF SQLITE3EXEPDB
+!IF $(FOR_WIN10)!=0
+SQLITE3EXEPDB =
+!ELSE
 SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 !ENDIF
+!ENDIF
+

 # These are the "standard" SQLite compilation options used when compiling for
 # the Windows platform.
 #
 !IFNDEF OPT_FEATURE_FLAGS
+!IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
+!ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF

+# Should the session extension be enabled?  If so, add compilation options
+# to enable it.
+#
+!IF $(SESSION)!=0
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1
+!ENDIF
+
 # These are the "extended" SQLite compilation options used when compiling for
 # the Windows 10 platform.
 #
@@ -275,6 +329,14 @@ PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\)
 CC = cl.exe
 !ENDIF

+# Check for the predefined command macro CSC.  This should point to a working
+# C Sharp compiler binary.  If it is not defined, simply define it to the
+# legacy default value 'csc.exe'.
+#
+!IFNDEF CSC
+CSC = csc.exe
+!ENDIF
+
 # Check for the command macro LD.  This should point to the linker binary for
 # the target platform.  If it is not defined, simply define it to the legacy
 # default value 'link.exe'.
@@ -399,18 +461,15 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
 TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
 !ENDIF

+# Check if warnings should be treated as errors when compiling.
+#
+!IF $(USE_FATAL_WARN)!=0
+TCC = $(TCC) -WX
+!ENDIF
+
 TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise
 RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)

-# Adjust the names of the primary targets for use with Windows 10.
-#
-!IF $(FOR_WIN10)!=0
-SQLITE3DLL = winsqlite3.dll
-SQLITE3LIB = winsqlite3.lib
-SQLITE3EXE = winsqlite3shell.exe
-SQLITE3EXEPDB =
-!ENDIF
-
 # Check if we want to use the "stdcall" calling convention when compiling.
 # This is not supported by the compilers for non-x86 platforms.  It should
 # also be noted here that building any target with these "stdcall" options
@@ -420,12 +479,12 @@ SQLITE3EXEPDB =
 #
 !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
 !IF "$(PLATFORM)"=="x86"
-CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
-SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
+CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
+SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
 !ELSE
 !IFNDEF PLATFORM
-CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
-SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
+CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
+SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
 !ELSE
 CORE_CCONV_OPTS =
 SHELL_CCONV_OPTS =
@@ -450,8 +509,10 @@ CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS)
 # when linking.
 #
 !IFNDEF CORE_LINK_DEP
-!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+!IF $(DYNAMIC_SHELL)!=0
 CORE_LINK_DEP =
+!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86"
+CORE_LINK_DEP = sqlite3.def
 !ELSE
 CORE_LINK_DEP =
 !ENDIF
@@ -460,8 +521,10 @@ CORE_LINK_DEP =
 # These are additional linker options used for the core library.
 #
 !IFNDEF CORE_LINK_OPTS
-!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+!IF $(DYNAMIC_SHELL)!=0
 CORE_LINK_OPTS =
+!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86"
+CORE_LINK_OPTS = /DEF:sqlite3.def
 !ELSE
 CORE_LINK_OPTS =
 !ENDIF
@@ -582,6 +645,10 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
 !IF $(DEBUG)>2
 TCC = $(TCC) -DSQLITE_DEBUG=1
 RCC = $(RCC) -DSQLITE_DEBUG=1
+!IF $(DYNAMIC_SHELL)==0
+TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
+RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
+!ENDIF
 !ENDIF

 !IF $(DEBUG)>4 || $(OSTRACE)!=0
@@ -707,6 +774,10 @@ RCC = $(RCC) -D_DEBUG
 !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
 TCC = $(TCC) -Od
 BCC = $(BCC) -Od
+!IF $(USE_RUNTIME_CHECKS)!=0
+TCC = $(TCC) -RTC1
+BCC = $(BCC) -RTC1
+!ENDIF
 !ELSEIF $(OPTIMIZATIONS)>=3
 TCC = $(TCC) -Ox
 BCC = $(BCC) -Ox
@@ -802,10 +873,10 @@ LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelH
 LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib
 !ENDIF

-# When compiling for UAP or the Windows 10 platform, some extra linker
+# When compiling for UWP or the Windows 10 platform, some extra linker
 # options are also required.
 #
-!IF $(FOR_UAP)!=0 || $(FOR_WIN10)!=0
+!IF $(FOR_UWP)!=0 || $(FOR_WIN10)!=0
 LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib
 LTLINKOPTS = $(LTLINKOPTS) mincore.lib
 !IFDEF PSDKLIBPATH
@@ -863,23 +934,28 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-all:	dll libsqlite3.lib shell
+all:	dll shell

 # Dynamic link library section.
 #
-dll: $(SQLITE3DLL)
+dll:	$(SQLITE3DLL)

 # Shell executable.
 #
-shell: $(SQLITE3EXE)
-
-libsqlite3.lib:	$(LIBOBJ)
-	$(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS)
+shell:	$(SQLITE3EXE)


-$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
+$(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
 	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)

+Replace.exe:
+	$(CSC) /target:exe $(TOP)\Replace.cs
+
+sqlite3.def:	Replace.exe $(LIBOBJ)
+	echo EXPORTS > sqlite3.def
+	dumpbin /all $(LIBOBJ) \
+		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
+		| sort >> sqlite3.def

 $(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
 	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
@@ -918,4 +994,5 @@ $(LIBRESOBJS):	$(TOP)\sqlite3.rc rcver.vc $(SQLITE3H)

 clean:
 	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
-	del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
+	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
+	del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL
@@ -6,8 +6,8 @@ This package contains:
 * the shell.c file used to build the sqlite3 command-line shell program
 * autoconf/automake installation infrastucture for building on POSIX
   compliant systems
- * a Makefile.msc and sqlite3.rc for building with Microsoft Visual C++ on
-   Windows
+ * a Makefile.msc, sqlite3.rc, and Replace.cs for building with Microsoft
+   Visual C++ on Windows

 SUMMARY OF HOW TO BUILD
 =======================
@@ -75,10 +75,10 @@ NMAKE command line as well:

  "NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86"

-Building for UAP 10.0
+Building for UWP 10.0
 ---------------------

-  FOR_WINRT=1 FOR_UAP=1
+  FOR_WINRT=1 FOR_UWP=1

 Using Microsoft Visual C++ 2015 (or later) is required.  When using the
 above, something like the following macros will need to be added to the
@@ -30,39 +30,52 @@ AC_FUNC_STRERROR_R
 AC_CONFIG_FILES([Makefile sqlite3.pc])
 AC_SUBST(BUILD_CFLAGS)

-#-----------------------------------------------------------------------
+#-------------------------------------------------------------------------
+# Two options to enable readline compatible libraries: 
+#
 #   --enable-editline
 #   --enable-readline
 #
-AC_ARG_ENABLE(editline, [AS_HELP_STRING(
-  [--enable-editline], 
-  [use BSD libedit])], 
-  [], [enable_editline=yes])
-AC_ARG_ENABLE(readline, [AS_HELP_STRING(
-  [--enable-readline], 
-  [use readline])], 
-  [], [enable_readline=no])
-if test x"$enable_editline" != xno ; then
-  sLIBS=$LIBS
-  LIBS=""
-  AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no])
-  READLINE_LIBS=$LIBS
-  if test x"$LIBS" != "x"; then
-     AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
-  else
-    unset ac_cv_search_readline
-  fi
-  LIBS=$sLIBS
-fi
-if test x"$enable_readline" != xno ; then
-  sLIBS=$LIBS
-  LIBS=""
-  AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], [])
-  AC_SEARCH_LIBS(readline, readline, [], [enable_readline=no])
-  AC_CHECK_FUNCS(readline, [], [])
-  READLINE_LIBS=$LIBS
-  LIBS=$sLIBS
-fi
+# Both are enabled by default. If, after command line processing both are
+# still enabled, the script searches for editline first and automatically
+# disables readline if it is found. So, to use readline explicitly, the
+# user must pass "--disable-editline". To disable command line editing
+# support altogether, "--disable-editline --disable-readline".
+#
+# When searching for either library, check for headers before libraries 
+# as some distros supply packages that contain libraries but not header
+# files, which come as a separate development package.
+#
+AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])])
+AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])])
+
+AS_IF([ test x"$enable_editline" != xno ],[
+  AC_CHECK_HEADERS([editline/readline.h],[
+    sLIBS=$LIBS
+    LIBS=""
+    AC_SEARCH_LIBS([readline],[edit],[
+      AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
+      READLINE_LIBS="$LIBS -ltinfo"
+      enable_readline=no
+    ],[],[-ltinfo])
+    AS_UNSET(ac_cv_search_readline)
+    LIBS=$sLIBS
+  ])
+])
+
+AS_IF([ test x"$enable_readline" != xno ],[
+  AC_CHECK_HEADERS([readline/readline.h],[
+    sLIBS=$LIBS
+    LIBS=""
+    AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], [])
+    AC_SEARCH_LIBS(readline,[readline edit], [
+      AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper)
+      READLINE_LIBS=$LIBS
+    ])
+    LIBS=$sLIBS
+  ])
+])
+
 AC_SUBST(READLINE_LIBS)
 #-----------------------------------------------------------------------

@@ -103,7 +116,7 @@ AC_SUBST(DYNAMIC_EXTENSION_FLAGS)
 AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
  [--enable-fts5], [include fts5 support [default=no]])], 
  [], [enable_fts5=no])
-if test x"$enable_fts5" == "xyes"; then
+if test x"$enable_fts5" = "xyes"; then
  AC_SEARCH_LIBS(log, m)
  FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
 fi
@@ -116,12 +129,24 @@ AC_SUBST(FTS5_FLAGS)
 AC_ARG_ENABLE(json1, [AS_HELP_STRING(
  [--enable-json1], [include json1 support [default=no]])], 
  [], [enable_json1=no])
-if test x"$enable_json1" == "xyes"; then
+if test x"$enable_json1" = "xyes"; then
  JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
 fi
 AC_SUBST(JSON1_FLAGS)
 #-----------------------------------------------------------------------

+#-----------------------------------------------------------------------
+#   --enable-session
+#
+AC_ARG_ENABLE(session, [AS_HELP_STRING(
+  [--enable-session], [enable the session extension [default=no]])], 
+  [], [enable_session=no])
+if test x"$enable_session" = "xyes"; then
+  SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
+fi
+AC_SUBST(SESSION_FLAGS)
+#-----------------------------------------------------------------------
+
 #-----------------------------------------------------------------------
 #   --enable-static-shell
 #
@@ -129,8 +154,8 @@ AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
  [--enable-static-shell], 
  [statically link libsqlite3 into shell tool [default=yes]])], 
  [], [enable_static_shell=yes])
-if test x"$enable_static_shell" == "xyes"; then
-  EXTRA_SHELL_OBJ=sqlite3.$OBJEXT
+if test x"$enable_static_shell" = "xyes"; then
+  EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT
 else
  EXTRA_SHELL_OBJ=libsqlite3.la
 fi
@@ -78,7 +78,6 @@ TEA_ADD_LIBS([])
 TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1])
 TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1])
 TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1])
-TEA_ADD_CFLAGS([-DSQLITE_OMIT_DEPRECATED=1])
 TEA_ADD_STUB_SOURCES([])
 TEA_ADD_TCL_SOURCES([])

@@ -45,6 +45,18 @@
 /* Define to 1 if you have the <memory.h> header file. */
 #undef HAVE_MEMORY_H

+/* Define to 1 if you have the pread() function. */
+#undef HAVE_PREAD
+
+/* Define to 1 if you have the pread64() function. */
+#undef HAVE_PREAD64
+
+/* Define to 1 if you have the pwrite() function. */
+#undef HAVE_PWRITE
+
+/* Define to 1 if you have the pwrite64() function. */
+#undef HAVE_PWRITE64
+
 /* Define to 1 if you have the <stdint.h> header file. */
 #undef HAVE_STDINT_H

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.11.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.20.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.11.0'
-PACKAGE_STRING='sqlite 3.11.0'
+PACKAGE_VERSION='3.20.0'
+PACKAGE_STRING='sqlite 3.20.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -903,11 +903,14 @@ with_readline_inc
 enable_debug
 enable_amalgamation
 enable_load_extension
+enable_memsys5
+enable_memsys3
 enable_fts3
 enable_fts4
 enable_fts5
 enable_json1
 enable_rtree
+enable_session
 enable_gcov
 '
      ac_precious_vars='build_alias
@@ -1460,7 +1463,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.11.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.20.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1525,7 +1528,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.11.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.20.0:";;
   esac
  cat <<\_ACEOF

@@ -1551,11 +1554,14 @@ Optional Features:
                          separately
  --disable-load-extension
                          Disable loading of external extensions
+  --enable-memsys5        Enable MEMSYS5
+  --enable-memsys3        Enable MEMSYS3
  --enable-fts3           Enable the FTS3 extension
  --enable-fts4           Enable the FTS4 extension
  --enable-fts5           Enable the FTS5 extension
  --enable-json1          Enable the JSON1 extension
  --enable-rtree          Enable the RTREE extension
+  --enable-session        Enable the SESSION extension
  --enable-gcov           Enable coverage testing using gcov

 Optional Packages:
@@ -1646,7 +1652,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.11.0
+sqlite configure 3.20.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2065,7 +2071,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.11.0, which was
+It was created by sqlite $as_me 3.20.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -3923,13 +3929,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
  lt_cv_nm_interface="BSD nm"
  echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3926: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3932: $ac_compile\"" >&5)
  (eval "$ac_compile" 2>conftest.err)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3929: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3932: output\"" >&5)
+  (eval echo "\"\$as_me:3938: output\"" >&5)
  cat conftest.out >&5
  if $GREP 'External.*some_variable' conftest.out > /dev/null; then
    lt_cv_nm_interface="MS dumpbin"
@@ -5135,7 +5141,7 @@ ia64-*-hpux*)
  ;;
 *-*-irix6*)
  # Find out which ABI we are using.
-  echo '#line 5138 "configure"' > conftest.$ac_ext
+  echo '#line 5144 "configure"' > conftest.$ac_ext
  if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  (eval $ac_compile) 2>&5
  ac_status=$?
@@ -6660,11 +6666,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6663: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6669: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:6667: \$? = $ac_status" >&5
+   echo "$as_me:6673: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -6999,11 +7005,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7002: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7008: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:7006: \$? = $ac_status" >&5
+   echo "$as_me:7012: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7104,11 +7110,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7107: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7113: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7111: \$? = $ac_status" >&5
+   echo "$as_me:7117: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -7159,11 +7165,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7162: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7168: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7166: \$? = $ac_status" >&5
+   echo "$as_me:7172: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -9539,7 +9545,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9542 "configure"
+#line 9548 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -9635,7 +9641,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9638 "configure"
+#line 9644 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -10273,7 +10279,7 @@ done
 #########
 # Figure out whether or not we have these functions
 #
-for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime
+for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64
 do :
  as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
 ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
@@ -10755,6 +10761,20 @@ else
      fi
    fi

+    # Recent versions of Xcode on Macs hid the tclConfig.sh file
+    # in a strange place.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
    # then check for a private Tcl installation
    if test x"${ac_cv_c_tclconfig}" = x ; then
      for i in \
@@ -11232,7 +11252,7 @@ else
 fi

 if test "${use_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1"
+  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 else
  TARGET_DEBUG="-DNDEBUG"
 fi
@@ -11323,6 +11343,44 @@ else
  OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
 fi

+##########
+# Do we want to support memsys3 and/or memsys5
+#
+# Check whether --enable-memsys5 was given.
+if test "${enable_memsys5+set}" = set; then :
+  enableval=$enable_memsys5; enable_memsys5=yes
+else
+  enable_memsys5=no
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5
+$as_echo_n "checking whether to support MEMSYS5... " >&6; }
+if test "${enable_memsys5}" = "yes"; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+# Check whether --enable-memsys3 was given.
+if test "${enable_memsys3+set}" = set; then :
+  enableval=$enable_memsys3; enable_memsys3=yes
+else
+  enable_memsys3=no
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5
+$as_echo_n "checking whether to support MEMSYS3... " >&6; }
+if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
 #########
 # See whether we should enable Full Text Search extensions
 # Check whether --enable-fts3 was given.
@@ -11333,7 +11391,7 @@ else
 fi

 if test "${enable_fts3}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
 fi
 # Check whether --enable-fts4 was given.
 if test "${enable_fts4+set}" = set; then :
@@ -11343,7 +11401,7 @@ else
 fi

 if test "${enable_fts4}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
 $as_echo_n "checking for library containing log... " >&6; }
 if ${ac_cv_search_log+:} false; then :
@@ -11409,7 +11467,7 @@ else
 fi

 if test "${enable_fts5}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
 $as_echo_n "checking for library containing log... " >&6; }
 if ${ac_cv_search_log+:} false; then :
@@ -11478,7 +11536,7 @@ else
 fi

 if test "${enable_json1}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
 fi

 #########
@@ -11491,11 +11549,25 @@ else
 fi

 if test "${enable_rtree}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
 fi

 #########
-# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
+# See whether we should enable the SESSION extension
+# Check whether --enable-session was given.
+if test "${enable_session+set}" = set; then :
+  enableval=$enable_session; enable_session=yes
+else
+  enable_session=no
+fi
+
+if test "${enable_session}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
+fi
+
+#########
+# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter
 for option in $CFLAGS $CPPFLAGS
 do
  case $option in
@@ -12079,7 +12151,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.11.0, which was
+This file was extended by sqlite $as_me 3.20.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12145,7 +12217,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.11.0
+sqlite config.status 3.20.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -108,7 +108,7 @@ AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h malloc.h])
 #########
 # Figure out whether or not we have these functions
 #
-AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime])
+AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64])

 #########
 # By default, we use the amalgamation (this may be changed below...)
@@ -334,6 +334,20 @@ if test "${use_tcl}" = "yes" ; then
      fi
    fi

+    # Recent versions of Xcode on Macs hid the tclConfig.sh file
+    # in a strange place.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
    # then check for a private Tcl installation
    if test x"${ac_cv_c_tclconfig}" = x ; then
      for i in \
@@ -546,7 +560,7 @@ AC_SEARCH_LIBS(fdatasync, [rt])
 AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
      [use_debug=$enableval],[use_debug=no])
 if test "${use_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1"
+  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 else
  TARGET_DEBUG="-DNDEBUG"
 fi
@@ -574,26 +588,50 @@ else
  OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
 fi

+##########
+# Do we want to support memsys3 and/or memsys5
+#
+AC_ARG_ENABLE(memsys5, 
+  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]),
+  [enable_memsys5=yes],[enable_memsys5=no])
+AC_MSG_CHECKING([whether to support MEMSYS5])
+if test "${enable_memsys5}" = "yes"; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
+  AC_MSG_RESULT([yes])
+else
+  AC_MSG_RESULT([no])
+fi
+AC_ARG_ENABLE(memsys3, 
+  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]),
+  [enable_memsys3=yes],[enable_memsys3=no])
+AC_MSG_CHECKING([whether to support MEMSYS3])
+if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
+  AC_MSG_RESULT([yes])
+else
+  AC_MSG_RESULT([no])
+fi
+
 #########
 # See whether we should enable Full Text Search extensions
 AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
      [Enable the FTS3 extension]),
      [enable_fts3=yes],[enable_fts3=no])
 if test "${enable_fts3}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
 fi
 AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
      [Enable the FTS4 extension]),
      [enable_fts4=yes],[enable_fts4=no])
 if test "${enable_fts4}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
  AC_SEARCH_LIBS([log],[m])
 fi
 AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
      [Enable the FTS5 extension]),
      [enable_fts5=yes],[enable_fts5=no])
 if test "${enable_fts5}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
  AC_SEARCH_LIBS([log],[m])
 fi

@@ -603,7 +641,7 @@ AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],
      [Enable the JSON1 extension]),
      [enable_json1=yes],[enable_json1=no])
 if test "${enable_json1}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
 fi

 #########
@@ -612,11 +650,21 @@ AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
      [Enable the RTREE extension]),
      [enable_rtree=yes],[enable_rtree=no])
 if test "${enable_rtree}" = "yes" ; then
-  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
 fi

 #########
-# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
+# See whether we should enable the SESSION extension
+AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
+      [Enable the SESSION extension]),
+      [enable_session=yes],[enable_session=no])
+if test "${enable_session}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
+fi
+
+#########
+# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter
 for option in $CFLAGS $CPPFLAGS
 do
  case $option in
@@ -5,14 +5,17 @@
 <body bgcolor=white>
 <h1 align=center>The Lemon Parser Generator</h1>

-<p>Lemon is an LALR(1) parser generator for C or C++.  
-It does the same job as ``bison'' and ``yacc''.
-But lemon is not another bison or yacc clone.  It
+<p>Lemon is an LALR(1) parser generator for C.
+It does the same job as "bison" and "yacc".
+But lemon is not a bison or yacc clone.  Lemon
 uses a different grammar syntax which is designed to
-reduce the number of coding errors.  Lemon also uses a more
-sophisticated parsing engine that is faster than yacc and
-bison and which is both reentrant and thread-safe.
-Furthermore, Lemon implements features that can be used
+reduce the number of coding errors.  Lemon also uses a
+parsing engine that is faster than yacc and
+bison and which is both reentrant and threadsafe.
+(Update: Since the previous sentence was written, bison
+has also been updated so that it too can generate a
+reentrant and threadsafe parser.)
+Lemon also implements features that can be used
 to eliminate resource leaks, making is suitable for use
 in long-running programs such as graphical user interfaces
 or embedded controllers.</p>
@@ -20,6 +23,26 @@ or embedded controllers.</p>
 <p>This document is an introduction to the Lemon
 parser generator.</p>

+<h2>Security Note</h2>
+
+<p>The language parser code created by Lemon is very robust and
+is well-suited for use in internet-facing applications that need to
+safely process maliciously crafted inputs.
+
+<p>The "lemon.exe" command-line tool itself works great when given a valid
+input grammar file and almost always gives helpful
+error messages for malformed inputs.  However,  it is possible for
+a malicious user to craft a grammar file that will cause 
+lemon.exe to crash.
+We do not see this as a problem, as lemon.exe is not intended to be used
+with hostile inputs.
+To summarize:</p>
+
+<ul>
+<li>Parser code generated by lemon &rarr; Robust and secure
+<li>The "lemon.exe" command line tool itself &rarr; Not so much
+</ul>
+
 <h2>Theory of Operation</h2>

 <p>The main goal of Lemon is to translate a context free grammar (CFG)
@@ -44,18 +67,18 @@ one and three files of outputs.
    automaton.
 </ul>
 By default, all three of these output files are generated.
-The header file is suppressed if the ``-m'' command-line option is
-used and the report file is omitted when ``-q'' is selected.</p>
+The header file is suppressed if the "-m" command-line option is
+used and the report file is omitted when "-q" is selected.</p>

-<p>The grammar specification file uses a ``.y'' suffix, by convention.
+<p>The grammar specification file uses a ".y" suffix, by convention.
 In the examples used in this document, we'll assume the name of the
-grammar file is ``gram.y''.  A typical use of Lemon would be the
+grammar file is "gram.y".  A typical use of Lemon would be the
 following command:
 <pre>
   lemon gram.y
 </pre>
-This command will generate three output files named ``gram.c'',
-``gram.h'' and ``gram.out''.
+This command will generate three output files named "gram.c",
+"gram.h" and "gram.out".
 The first is C code to implement the parser.  The second
 is the header file that defines numerical values for all
 terminal symbols, and the last is the report that explains
@@ -71,39 +94,35 @@ with a brief explanation of what each does by typing
 </pre>
 As of this writing, the following command-line options are supported:
 <ul>
-<li><tt>-b</tt>
-<li><tt>-c</tt>
-<li><tt>-g</tt>
-<li><tt>-m</tt>
-<li><tt>-q</tt>
-<li><tt>-s</tt>
-<li><tt>-x</tt>
+<li><b>-b</b>
+Show only the basis for each parser state in the report file.
+<li><b>-c</b>
+Do not compress the generated action tables.
+<li><b>-D<i>name</i></b>
+Define C preprocessor macro <i>name</i>.  This macro is useable by
+"%ifdef" lines in the grammar file.
+<li><b>-g</b>
+Do not generate a parser.  Instead write the input grammar to standard
+output with all comments, actions, and other extraneous text removed.
+<li><b>-l</b>
+Omit "#line" directives in the generated parser C code.
+<li><b>-m</b>
+Cause the output C source code to be compatible with the "makeheaders"
+program. 
+<li><b>-p</b>
+Display all conflicts that are resolved by 
+<a href='#precrules'>precedence rules</a>.
+<li><b>-q</b>
+Suppress generation of the report file.
+<li><b>-r</b>
+Do not sort or renumber the parser states as part of optimization.
+<li><b>-s</b>
+Show parser statistics before existing.
+<li><b>-T<i>file</i></b>
+Use <i>file</i> as the template for the generated C-code parser implementation.
+<li><b>-x</b>
+Print the Lemon version number.
 </ul>
-The ``-b'' option reduces the amount of text in the report file by
-printing only the basis of each parser state, rather than the full
-configuration.
-The ``-c'' option suppresses action table compression.  Using -c
-will make the parser a little larger and slower but it will detect
-syntax errors sooner.
-The ``-g'' option causes no output files to be generated at all.
-Instead, the input grammar file is printed on standard output but
-with all comments, actions and other extraneous text deleted.  This
-is a useful way to get a quick summary of a grammar.
-The ``-m'' option causes the output C source file to be compatible
-with the ``makeheaders'' program.
-Makeheaders is a program that automatically generates header files
-from C source code.  When the ``-m'' option is used, the header
-file is not output since the makeheaders program will take care
-of generated all header files automatically.
-The ``-q'' option suppresses the report file.
-Using ``-s'' causes a brief summary of parser statistics to be
-printed.  Like this:
-<pre>
-   Parser statistics: 74 terminals, 70 nonterminals, 179 rules
-                      340 states, 2026 parser table entries, 0 conflicts
-</pre>
-Finally, the ``-x'' option causes Lemon to print its version number
-and then stops without attempting to read the grammar or generate a parser.</p>

 <h3>The Parser Interface</h3>

@@ -121,12 +140,12 @@ A new parser is created as follows:
 </pre>
 The ParseAlloc() routine allocates and initializes a new parser and
 returns a pointer to it.
-The actual data structure used to represent a parser is opaque --
+The actual data structure used to represent a parser is opaque &mdash;
 its internal structure is not visible or usable by the calling routine.
 For this reason, the ParseAlloc() routine returns a pointer to void
 rather than a pointer to some particular structure.
 The sole argument to the ParseAlloc() routine is a pointer to the
-subroutine used to allocate memory.  Typically this means ``malloc()''.</p>
+subroutine used to allocate memory.  Typically this means malloc().</p>

 <p>After a program is finished using a parser, it can reclaim all
 memory allocated by that parser by calling
@@ -151,18 +170,19 @@ type of the next token in the data stream.
 There is one token type for each terminal symbol in the grammar.
 The gram.h file generated by Lemon contains #define statements that
 map symbolic terminal symbol names into appropriate integer values.
-(A value of 0 for the second argument is a special flag to the
-parser to indicate that the end of input has been reached.)
+A value of 0 for the second argument is a special flag to the
+parser to indicate that the end of input has been reached.
 The third argument is the value of the given token.  By default,
 the type of the third argument is integer, but the grammar will
 usually redefine this type to be some kind of structure.
 Typically the second argument will be a broad category of tokens
-such as ``identifier'' or ``number'' and the third argument will
+such as "identifier" or "number" and the third argument will
 be the name of the identifier or the value of the number.</p>

 <p>The Parse() function may have either three or four arguments,
-depending on the grammar.  If the grammar specification file request
-it, the Parse() function will have a fourth parameter that can be
+depending on the grammar.  If the grammar specification file requests
+it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>),
+the Parse() function will have a fourth parameter that can be
 of any type chosen by the programmer.  The parser doesn't do anything
 with this argument except to pass it through to action routines.
 This is a convenient mechanism for passing state information down
@@ -192,7 +212,7 @@ following:
 </pre>
 This example shows a user-written routine that parses a file of
 text and returns a pointer to the parse tree.
-(We've omitted all error-handling from this example to keep it
+(All error-handling code is omitted from this example to keep it
 simple.)
 We assume the existence of some kind of tokenizer which is created
 using TokenizerCreate() on line 8 and deleted by TokenizerFree()
@@ -263,6 +283,12 @@ with prior yacc and bison experience.
 But after years of experience using Lemon, I firmly
 believe that the Lemon way of doing things is better.</p>

+<p><i>Updated as of 2016-02-16:</i>
+The text above was written in the 1990s.
+We are told that Bison has lately been enhanced to support the
+tokenizer-calls-parser paradigm used by Lemon, and to obviate the
+need for global variables.</p>
+
 <h2>Input File Syntax</h2>

 <p>The main purpose of the grammar specification file for Lemon is
@@ -280,7 +306,7 @@ tokens) and it honors the same commenting conventions as C and C++.</p>
 <h3>Terminals and Nonterminals</h3>

 <p>A terminal symbol (token) is any string of alphanumeric
-and underscore characters
+and/or underscore characters
 that begins with an upper case letter.
 A terminal can contain lowercase letters after the first character,
 but the usual convention is to make terminals all upper case.
@@ -307,7 +333,7 @@ must have alphanumeric names.</p>
 <p>The main component of a Lemon grammar file is a sequence of grammar
 rules.
 Each grammar rule consists of a nonterminal symbol followed by
-the special symbol ``::='' and then a list of terminals and/or nonterminals.
+the special symbol "::=" and then a list of terminals and/or nonterminals.
 The rule is terminated by a period.
 The list of terminals and nonterminals on the right-hand side of the
 rule can be empty.
@@ -323,9 +349,9 @@ A typical sequence of grammar rules might look something like this:
 </pre>
 </p>

-<p>There is one non-terminal in this example, ``expr'', and five
-terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'',
-``RPAREN'' and ``VALUE''.</p>
+<p>There is one non-terminal in this example, "expr", and five
+terminal symbols or tokens: "PLUS", "TIMES", "LPAREN",
+"RPAREN" and "VALUE".</p>

 <p>Like yacc and bison, Lemon allows the grammar to specify a block
 of C code that will be executed whenever a grammar rule is reduced
@@ -341,15 +367,15 @@ For example:

 <p>In order to be useful, grammar actions must normally be linked to
 their associated grammar rules.
-In yacc and bison, this is accomplished by embedding a ``$$'' in the
+In yacc and bison, this is accomplished by embedding a "$$" in the
 action to stand for the value of the left-hand side of the rule and
-symbols ``$1'', ``$2'', and so forth to stand for the value of
+symbols "$1", "$2", and so forth to stand for the value of
 the terminal or nonterminal at position 1, 2 and so forth on the
 right-hand side of the rule.
 This idea is very powerful, but it is also very error-prone.  The
 single most common source of errors in a yacc or bison grammar is
 to miscount the number of symbols on the right-hand side of a grammar
-rule and say ``$7'' when you really mean ``$8''.</p>
+rule and say "$7" when you really mean "$8".</p>

 <p>Lemon avoids the need to count grammar symbols by assigning symbolic
 names to each symbol in a grammar rule and then using those symbolic
@@ -379,7 +405,7 @@ For example, the rule
 <pre>
  expr(A) ::= expr(B) PLUS expr(C).  { A = B; }
 </pre>
-will generate an error because the linking symbol ``C'' is used
+will generate an error because the linking symbol "C" is used
 in the grammar rule but not in the reduce action.</p>

 <p>The Lemon notation for linking grammar rules to reduce actions
@@ -387,6 +413,7 @@ also facilitates the use of destructors for reclaiming memory
 allocated by the values of terminals and nonterminals on the
 right-hand side of a rule.</p>

+<a name='precrules'></a>
 <h3>Precedence Rules</h3>

 <p>Lemon resolves parsing ambiguities in exactly the same way as
@@ -398,7 +425,10 @@ whichever rule comes first in the grammar file.</p>
 yacc and bison, Lemon allows a measure of control 
 over the resolution of paring conflicts using precedence rules.
 A precedence value can be assigned to any terminal symbol
-using the %left, %right or %nonassoc directives.  Terminal symbols
+using the 
+<a href='#pleft'>%left</a>,
+<a href='#pright'>%right</a> or
+<a href='#pnonassoc'>%nonassoc</a> directives.  Terminal symbols
 mentioned in earlier directives have a lower precedence that
 terminal symbols mentioned in later directives.  For example:</p>

@@ -518,7 +548,11 @@ other than that, the order of directives in Lemon is arbitrary.</p>
 <li><tt>%default_destructor</tt>
 <li><tt>%default_type</tt>
 <li><tt>%destructor</tt>
+<li><tt>%endif</tt>
 <li><tt>%extra_argument</tt>
+<li><tt>%fallback</tt>
+<li><tt>%ifdef</tt>
+<li><tt>%ifndef</tt>
 <li><tt>%include</tt>
 <li><tt>%left</tt>
 <li><tt>%name</tt>
@@ -530,49 +564,57 @@ other than that, the order of directives in Lemon is arbitrary.</p>
 <li><tt>%stack_size</tt>
 <li><tt>%start_symbol</tt>
 <li><tt>%syntax_error</tt>
+<li><tt>%token_class</tt>
 <li><tt>%token_destructor</tt>
 <li><tt>%token_prefix</tt>
 <li><tt>%token_type</tt>
 <li><tt>%type</tt>
+<li><tt>%wildcard</tt>
 </ul>
 Each of these directives will be described separately in the
 following sections:</p>

+<a name='pcode'></a>
 <h4>The <tt>%code</tt> directive</h4>

-<p>The %code directive is used to specify addition C/C++ code that
+<p>The %code directive is used to specify addition C code that
 is added to the end of the main output file.  This is similar to
-the %include directive except that %include is inserted at the
-beginning of the main output file.</p>
+the <a href='#pinclude'>%include</a> directive except that %include
+is inserted at the beginning of the main output file.</p>

 <p>%code is typically used to include some action routines or perhaps
-a tokenizer as part of the output file.</p>
+a tokenizer or even the "main()" function 
+as part of the output file.</p>

+<a name='default_destructor'></a>
 <h4>The <tt>%default_destructor</tt> directive</h4>

 <p>The %default_destructor directive specifies a destructor to 
 use for non-terminals that do not have their own destructor
 specified by a separate %destructor directive.  See the documentation
-on the %destructor directive below for additional information.</p>
+on the <a name='#destructor'>%destructor</a> directive below for
+additional information.</p>

 <p>In some grammers, many different non-terminal symbols have the
 same datatype and hence the same destructor.  This directive is
 a convenience way to specify the same destructor for all those
 non-terminals using a single statement.</p>

+<a name='default_type'></a>
 <h4>The <tt>%default_type</tt> directive</h4>

 <p>The %default_type directive specifies the datatype of non-terminal
 symbols that do no have their own datatype defined using a separate
-%type directive.  See the documentation on %type below for addition
-information.</p>
+<a href='#ptype'>%type</a> directive.  
+</p>

+<a name='destructor'></a>
 <h4>The <tt>%destructor</tt> directive</h4>

 <p>The %destructor directive is used to specify a destructor for
 a non-terminal symbol.
-(See also the %token_destructor directive which is used to
-specify a destructor for terminal symbols.)</p>
+(See also the <a href='#token_destructor'>%token_destructor</a>
+directive which is used to specify a destructor for terminal symbols.)</p>

 <p>A non-terminal's destructor is called to dispose of the
 non-terminal's value whenever the non-terminal is popped from
@@ -595,28 +637,28 @@ or other resources held by that non-terminal.</p>
 </pre>
 This example is a bit contrived but it serves to illustrate how
 destructors work.  The example shows a non-terminal named
-``nt'' that holds values of type ``void*''.  When the rule for
-an ``nt'' reduces, it sets the value of the non-terminal to
+"nt" that holds values of type "void*".  When the rule for
+an "nt" reduces, it sets the value of the non-terminal to
 space obtained from malloc().  Later, when the nt non-terminal
 is popped from the stack, the destructor will fire and call
 free() on this malloced space, thus avoiding a memory leak.
-(Note that the symbol ``$$'' in the destructor code is replaced
+(Note that the symbol "$$" in the destructor code is replaced
 by the value of the non-terminal.)</p>

 <p>It is important to note that the value of a non-terminal is passed
 to the destructor whenever the non-terminal is removed from the
 stack, unless the non-terminal is used in a C-code action.  If
 the non-terminal is used by C-code, then it is assumed that the
-C-code will take care of destroying it if it should really
-be destroyed.  More commonly, the value is used to build some
+C-code will take care of destroying it.
+More commonly, the value is used to build some
 larger structure and we don't want to destroy it, which is why
 the destructor is not called in this circumstance.</p>

-<p>By appropriate use of destructors, it is possible to
-build a parser using Lemon that can be used within a long-running
-program, such as a GUI, that will not leak memory or other resources.
+<p>Destructors help avoid memory leaks by automatically freeing
+allocated objects when they go out of scope.
 To do the same using yacc or bison is much more difficult.</p>

+<a name="extraarg"></a>
 <h4>The <tt>%extra_argument</tt> directive</h4>

 The %extra_argument directive instructs Lemon to add a 4th parameter
@@ -630,17 +672,66 @@ and so forth.  For example, if the grammar file contains:</p>
 </pre></p>

 <p>Then the Parse() function generated will have an 4th parameter
-of type ``MyStruct*'' and all action routines will have access to
-a variable named ``pAbc'' that is the value of the 4th parameter
+of type "MyStruct*" and all action routines will have access to
+a variable named "pAbc" that is the value of the 4th parameter
 in the most recent call to Parse().</p>

+<a name='pfallback'></a>
+<h4>The <tt>%fallback</tt> directive</h4>
+
+<p>The %fallback directive specifies an alternative meaning for one
+or more tokens.  The alternative meaning is tried if the original token
+would have generated a syntax error.
+
+<p>The %fallback directive was added to support robust parsing of SQL
+syntax in <a href="https://www.sqlite.org/">SQLite</a>.
+The SQL language contains a large assortment of keywords, each of which
+appears as a different token to the language parser.  SQL contains so
+many keywords, that it can be difficult for programmers to keep up with
+them all.  Programmers will, therefore, sometimes mistakenly use an
+obscure language keyword for an identifier.  The %fallback directive
+provides a mechanism to tell the parser:  "If you are unable to parse
+this keyword, try treating it as an identifier instead."
+
+<p>The syntax of %fallback is as follows:
+
+<blockquote>
+<tt>%fallback</tt>  <i>ID</i> <i>TOKEN...</i> <b>.</b>
+</blockquote>
+
+<p>In words, the %fallback directive is followed by a list of token names
+terminated by a period.  The first token name is the fallback token - the
+token to which all the other tokens fall back to.  The second and subsequent
+arguments are tokens which fall back to the token identified by the first
+argument.
+
+<a name='pifdef'></a>
+<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4>
+
+<p>The %ifdef, %ifndef, and %endif directives are similar to
+#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general.
+Each of these directives must begin at the left margin.  No whitespace
+is allowed between the "%" and the directive name.
+
+<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is
+ignored unless the "-DMACRO" command-line option is used.  Grammar text
+betwen "%ifndef MACRO" and the next nested "%endif" is included except when
+the "-DMACRO" command-line option is used.
+
+<p>Note that the argument to %ifdef and %ifndef must be a single 
+preprocessor symbol name, not a general expression.  There is no "%else"
+directive.
+
+
+<a name='pinclude'></a>
 <h4>The <tt>%include</tt> directive</h4>

 <p>The %include directive specifies C code that is included at the
 top of the generated parser.  You can include any text you want --
 the Lemon parser generator copies it blindly.  If you have multiple
-%include directives in your grammar file the value of the last
-%include directive overwrites all the others.</p.
+%include directives in your grammar file, their values are concatenated
+so that all %include code ultimately appears near the top of the
+generated parser, in the same order as it appeared in the grammer.</p>

 <p>The %include directive is very handy for getting some extra #include
 preprocessor statements at the beginning of the generated parser.
@@ -653,12 +744,13 @@ For example:</p>
 <p>This might be needed, for example, if some of the C actions in the
 grammar call functions that are prototyed in unistd.h.</p>

+<a name='pleft'></a>
 <h4>The <tt>%left</tt> directive</h4>

-The %left directive is used (along with the %right and
-%nonassoc directives) to declare precedences of terminal
-symbols.  Every terminal symbol whose name appears after
-a %left directive but before the next period (``.'') is
+The %left directive is used (along with the <a href='#pright'>%right</a> and
+<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of 
+terminal symbols.  Every terminal symbol whose name appears after
+a %left directive but before the next period (".") is
 given the same left-associative precedence value.  Subsequent
 %left directives have higher precedence.  For example:</p>

@@ -679,10 +771,11 @@ a large amount of stack space if you make heavy use or right-associative
 operators.  For this reason, it is recommended that you use %left
 rather than %right whenever possible.</p>

+<a name='pname'></a>
 <h4>The <tt>%name</tt> directive</h4>

 <p>By default, the functions generated by Lemon all begin with the
-five-character string ``Parse''.  You can change this string to something
+five-character string "Parse".  You can change this string to something
 different using the %name directive.  For instance:</p>

 <p><pre>
@@ -701,16 +794,19 @@ The %name directive allows you to generator two or more different
 parsers and link them all into the same executable.
 </p>

+<a name='pnonassoc'></a>
 <h4>The <tt>%nonassoc</tt> directive</h4>

 <p>This directive is used to assign non-associative precedence to
-one or more terminal symbols.  See the section on precedence rules
-or on the %left directive for additional information.</p>
+one or more terminal symbols.  See the section on 
+<a href='#precrules'>precedence rules</a>
+or on the <a href='#pleft'>%left</a> directive for additional information.</p>

+<a name='parse_accept'></a>
 <h4>The <tt>%parse_accept</tt> directive</h4>

 <p>The %parse_accept directive specifies a block of C code that is
-executed whenever the parser accepts its input string.  To ``accept''
+executed whenever the parser accepts its input string.  To "accept"
 an input string means that the parser was able to process all tokens
 without error.</p>

@@ -722,7 +818,7 @@ without error.</p>
   }
 </pre></p>

-
+<a name='parse_failure'></a>
 <h4>The <tt>%parse_failure</tt> directive</h4>

 <p>The %parse_failure directive specifies a block of C code that
@@ -737,12 +833,15 @@ only invoked when parsing is unable to continue.</p>
   }
 </pre></p>

+<a name='pright'></a>
 <h4>The <tt>%right</tt> directive</h4>

 <p>This directive is used to assign right-associative precedence to
-one or more terminal symbols.  See the section on precedence rules
-or on the %left directive for additional information.</p>
+one or more terminal symbols.  See the section on 
+<a href='#precrules'>precedence rules</a>
+or on the <a href='#pleft'>%left</a> directive for additional information.</p>

+<a name='stack_overflow'></a>
 <h4>The <tt>%stack_overflow</tt> directive</h4>

 <p>The %stack_overflow directive specifies a block of C code that
@@ -771,6 +870,7 @@ Not like this:
   list ::= .
 </pre>

+<a name='stack_size'></a>
 <h4>The <tt>%stack_size</tt> directive</h4>

 <p>If stack overflow is a problem and you can't resolve the trouble
@@ -783,6 +883,7 @@ with a stack of the requested size.  The default value is 100.</p>
   %stack_size 2000
 </pre></p>

+<a name='start_symbol'></a>
 <h4>The <tt>%start_symbol</tt> directive</h4>

 <p>By default, the start-symbol for the grammar that Lemon generates
@@ -793,6 +894,7 @@ can choose a different start-symbol using the %start_symbol directive.</p>
   %start_symbol  prog
 </pre></p>

+<a name='token_destructor'></a>
 <h4>The <tt>%token_destructor</tt> directive</h4>

 <p>The %destructor directive assigns a destructor to a non-terminal
@@ -805,6 +907,7 @@ the %token_type directive) and so they use a common destructor.  Other
 than that, the token destructor works just like the non-terminal
 destructors.</p>

+<a name='token_prefix'></a>
 <h4>The <tt>%token_prefix</tt> directive</h4>

 <p>Lemon generates #defines that assign small integer constants
@@ -830,6 +933,7 @@ to cause Lemon to produce these symbols instead:
    #define TOKEN_PLUS       4
 </pre>

+<a name='token_type'></a><a name='ptype'></a>
 <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>

 <p>These directives are used to specify the data types for values
@@ -845,7 +949,7 @@ token structure.  Like this:</p>
 </pre></p>

 <p>If the data type of terminals is not specified, the default value
-is ``int''.</p>
+is "void*".</p>

 <p>Non-terminal symbols can each have their own data types.  Typically
 the data type  of a non-terminal is a pointer to the root of a parse-tree
@@ -866,6 +970,17 @@ non-terminal whose data type requires 1K of storage, then your 100
 entry parser stack will require 100K of heap space.  If you are willing
 and able to pay that price, fine.  You just need to know.</p>

+<a name='pwildcard'></a>
+<h4>The <tt>%wildcard</tt> directive</h4>
+
+<p>The %wildcard directive is followed by a single token name and a
+period.  This directive specifies that the identified token should 
+match any input token.
+
+<p>When the generated parser has the choice of matching an input against
+the wildcard token and some other token, the other token is always used.
+The wildcard token is only matched if there are no other alternatives.
+
 <h3>Error Processing</h3>

 <p>After extensive experimentation over several years, it has been
@@ -877,7 +992,7 @@ first invokes the code specified by the %syntax_error directive, if
 any.  It then enters its error recovery strategy.  The error recovery
 strategy is to begin popping the parsers stack until it enters a
 state where it is permitted to shift a special non-terminal symbol
-named ``error''.  It then shifts this non-terminal and continues
+named "error".  It then shifts this non-terminal and continues
 parsing.  But the %syntax_error routine will not be called again
 until at least three new tokens have been successfully shifted.</p>

@@ -886,7 +1001,7 @@ is unable to shift the error symbol, then the %parse_failed routine
 is invoked and the parser resets itself to its start state, ready
 to begin parsing a new file.  This is what will happen at the very
 first syntax error, of course, if there are no instances of the 
-``error'' non-terminal in your grammar.</p>
+"error" non-terminal in your grammar.</p>

 </body>
 </html>
@@ -0,0 +1,8 @@
+## Loadable Extensions
+
+Various [loadable extensions](https://www.sqlite.org/loadext.html) for
+SQLite are found in subfolders.
+
+Most subfolders are dedicated to a single loadable extension (for
+example FTS5, or RTREE).  But the misc/ subfolder contains a collection
+of smaller single-file extensions.
@@ -1,2 +0,0 @@
-Version loadable extensions to SQLite are found in subfolders
-of this folder.
@@ -99,7 +99,11 @@ static void scalarFunc(

 #ifdef SQLITE_TEST

-#include <tcl.h>
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include <string.h>

 /*
@@ -349,8 +349,9 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
 ** Return the number of bytes read, or 0 on error.
 ** The value is stored in *v.
 */
-int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
-  const char *pStart = p;
+int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
+  const unsigned char *p = (const unsigned char*)pBuf;
+  const unsigned char *pStart = p;
  u32 a;
  u64 b;
  int shift;
@@ -371,8 +372,8 @@ int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
 }

 /*
-** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
-** 32-bit integer before it is returned.
+** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to 
+** a non-negative 32-bit integer before it is returned.
 */
 int sqlite3Fts3GetVarint32(const char *p, int *pi){
  u32 a;
@@ -388,7 +389,9 @@ int sqlite3Fts3GetVarint32(const char *p, int *pi){
  GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *pi, 3);
  GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4);
  a = (a & 0x0FFFFFFF );
-  *pi = (int)(a | ((u32)(*p & 0x0F) << 28));
+  *pi = (int)(a | ((u32)(*p & 0x07) << 28));
+  assert( 0==(a & 0x80000000) );
+  assert( *pi>=0 );
  return 5;
 }

@@ -492,6 +495,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
  assert( p->pSegments==0 );

  /* Free any prepared statements held */
+  sqlite3_finalize(p->pSeekStmt);
  for(i=0; i<SizeofArray(p->aStmt); i++){
    sqlite3_finalize(p->aStmt[i]);
  }
@@ -1217,65 +1221,66 @@ static int fts3InitVtab(
            break;
          }
        }
-        if( iOpt==SizeofArray(aFts4Opt) ){
-          sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
-          rc = SQLITE_ERROR;
-        }else{
-          switch( iOpt ){
-            case 0:               /* MATCHINFO */
-              if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
-                sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
-                rc = SQLITE_ERROR;
-              }
-              bNoDocsize = 1;
-              break;
+        switch( iOpt ){
+          case 0:               /* MATCHINFO */
+            if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
+              sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
+              rc = SQLITE_ERROR;
+            }
+            bNoDocsize = 1;
+            break;

-            case 1:               /* PREFIX */
-              sqlite3_free(zPrefix);
-              zPrefix = zVal;
-              zVal = 0;
-              break;
+          case 1:               /* PREFIX */
+            sqlite3_free(zPrefix);
+            zPrefix = zVal;
+            zVal = 0;
+            break;

-            case 2:               /* COMPRESS */
-              sqlite3_free(zCompress);
-              zCompress = zVal;
-              zVal = 0;
-              break;
+          case 2:               /* COMPRESS */
+            sqlite3_free(zCompress);
+            zCompress = zVal;
+            zVal = 0;
+            break;

-            case 3:               /* UNCOMPRESS */
-              sqlite3_free(zUncompress);
-              zUncompress = zVal;
-              zVal = 0;
-              break;
+          case 3:               /* UNCOMPRESS */
+            sqlite3_free(zUncompress);
+            zUncompress = zVal;
+            zVal = 0;
+            break;

-            case 4:               /* ORDER */
-              if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
-               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
-              ){
-                sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
-                rc = SQLITE_ERROR;
-              }
-              bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
-              break;
+          case 4:               /* ORDER */
+            if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
+             && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
+            ){
+              sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
+              rc = SQLITE_ERROR;
+            }
+            bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
+            break;

-            case 5:              /* CONTENT */
-              sqlite3_free(zContent);
-              zContent = zVal;
-              zVal = 0;
-              break;
+          case 5:              /* CONTENT */
+            sqlite3_free(zContent);
+            zContent = zVal;
+            zVal = 0;
+            break;

-            case 6:              /* LANGUAGEID */
-              assert( iOpt==6 );
-              sqlite3_free(zLanguageid);
-              zLanguageid = zVal;
-              zVal = 0;
-              break;
+          case 6:              /* LANGUAGEID */
+            assert( iOpt==6 );
+            sqlite3_free(zLanguageid);
+            zLanguageid = zVal;
+            zVal = 0;
+            break;

-            case 7:              /* NOTINDEXED */
-              azNotindexed[nNotindexed++] = zVal;
-              zVal = 0;
-              break;
-          }
+          case 7:              /* NOTINDEXED */
+            azNotindexed[nNotindexed++] = zVal;
+            zVal = 0;
+            break;
+
+          default:
+            assert( iOpt==SizeofArray(aFts4Opt) );
+            sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
+            rc = SQLITE_ERROR;
+            break;
        }
        sqlite3_free(zVal);
      }
@@ -1363,9 +1368,9 @@ static int fts3InitVtab(
  p->pTokenizer = pTokenizer;
  p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  p->bHasDocsize = (isFts4 && bNoDocsize==0);
-  p->bHasStat = isFts4;
-  p->bFts4 = isFts4;
-  p->bDescIdx = bDescIdx;
+  p->bHasStat = (u8)isFts4;
+  p->bFts4 = (u8)isFts4;
+  p->bDescIdx = (u8)bDescIdx;
  p->nAutoincrmerge = 0xff;   /* 0xff means setting unknown */
  p->zContentTbl = zContent;
  p->zLanguageid = zLanguageid;
@@ -1396,7 +1401,9 @@ static int fts3InitVtab(
    char *z; 
    int n = 0;
    z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n);
-    memcpy(zCsr, z, n);
+    if( n>0 ){
+      memcpy(zCsr, z, n);
+    }
    zCsr[n] = '\0';
    sqlite3Fts3Dequote(zCsr);
    p->azColumn[iCol] = zCsr;
@@ -1680,6 +1687,39 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
  return SQLITE_OK;
 }

+/*
+** Finalize the statement handle at pCsr->pStmt.
+**
+** Or, if that statement handle is one created by fts3CursorSeekStmt(),
+** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement
+** pointer there instead of finalizing it.
+*/
+static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){
+  if( pCsr->bSeekStmt ){
+    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+    if( p->pSeekStmt==0 ){
+      p->pSeekStmt = pCsr->pStmt;
+      sqlite3_reset(pCsr->pStmt);
+      pCsr->pStmt = 0;
+    }
+    pCsr->bSeekStmt = 0;
+  }
+  sqlite3_finalize(pCsr->pStmt);
+}
+
+/*
+** Free all resources currently held by the cursor passed as the only
+** argument.
+*/
+static void fts3ClearCursor(Fts3Cursor *pCsr){
+  fts3CursorFinalizeStmt(pCsr);
+  sqlite3Fts3FreeDeferredTokens(pCsr);
+  sqlite3_free(pCsr->aDoclist);
+  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
+  sqlite3Fts3ExprFree(pCsr->pExpr);
+  memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
+}
+
 /*
 ** Close the cursor.  For additional information see the documentation
 ** on the xClose method of the virtual table interface.
@@ -1687,11 +1727,7 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
 static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
  Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
-  sqlite3_finalize(pCsr->pStmt);
-  sqlite3Fts3ExprFree(pCsr->pExpr);
-  sqlite3Fts3FreeDeferredTokens(pCsr);
-  sqlite3_free(pCsr->aDoclist);
-  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
+  fts3ClearCursor(pCsr);
  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  sqlite3_free(pCsr);
  return SQLITE_OK;
@@ -1705,20 +1741,23 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
 **
 ** (or the equivalent for a content=xxx table) and set pCsr->pStmt to
 ** it. If an error occurs, return an SQLite error code.
-**
-** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK.
 */
-static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
+static int fts3CursorSeekStmt(Fts3Cursor *pCsr){
  int rc = SQLITE_OK;
  if( pCsr->pStmt==0 ){
    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
    char *zSql;
-    zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
-    if( !zSql ) return SQLITE_NOMEM;
-    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
-    sqlite3_free(zSql);
+    if( p->pSeekStmt ){
+      pCsr->pStmt = p->pSeekStmt;
+      p->pSeekStmt = 0;
+    }else{
+      zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
+      if( !zSql ) return SQLITE_NOMEM;
+      rc = sqlite3_prepare_v3(p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
+      sqlite3_free(zSql);
+    }
+    if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1;
  }
-  *ppStmt = pCsr->pStmt;
  return rc;
 }

@@ -1730,9 +1769,7 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
 static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
  int rc = SQLITE_OK;
  if( pCsr->isRequireSeek ){
-    sqlite3_stmt *pStmt = 0;
-
-    rc = fts3CursorSeekStmt(pCsr, &pStmt);
+    rc = fts3CursorSeekStmt(pCsr);
    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
      pCsr->isRequireSeek = 0;
@@ -1821,7 +1858,8 @@ static int fts3ScanInteriorNode(
    isFirstTerm = 0;
    zCsr += fts3GetVarint32(zCsr, &nSuffix);
    
-    if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
+    assert( nPrefix>=0 && nSuffix>=0 );
+    if( &zCsr[nSuffix]>zEnd ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
@@ -2631,7 +2669,7 @@ int sqlite3Fts3FirstFilter(
    fts3ColumnlistCopy(0, &p);
  }

-  while( p<pEnd && *p==0x01 ){
+  while( p<pEnd ){
    sqlite3_int64 iCol;
    p++;
    p += sqlite3Fts3GetVarint(p, &iCol);
@@ -3190,11 +3228,7 @@ static int fts3FilterMethod(
  assert( iIdx==nVal );

  /* In case the cursor has been used before, clear it now. */
-  sqlite3_finalize(pCsr->pStmt);
-  sqlite3_free(pCsr->aDoclist);
-  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
-  sqlite3Fts3ExprFree(pCsr->pExpr);
-  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
+  fts3ClearCursor(pCsr);

  /* Set the lower and upper bounds on docids to return */
  pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
@@ -3252,13 +3286,13 @@ static int fts3FilterMethod(
      );
    }
    if( zSql ){
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+      rc = sqlite3_prepare_v3(p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
      sqlite3_free(zSql);
    }else{
      rc = SQLITE_NOMEM;
    }
  }else if( eSearch==FTS3_DOCID_SEARCH ){
-    rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt);
+    rc = fts3CursorSeekStmt(pCsr);
    if( rc==SQLITE_OK ){
      rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons);
    }
@@ -3273,7 +3307,12 @@ static int fts3FilterMethod(
 ** routine to find out if it has reached the end of a result set.
 */
 static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
-  return ((Fts3Cursor *)pCursor)->isEof;
+  Fts3Cursor *pCsr = (Fts3Cursor*)pCursor;
+  if( pCsr->isEof ){
+    fts3ClearCursor(pCsr);
+    pCsr->isEof = 1;
+  }
+  return pCsr->isEof;
 }

 /* 
@@ -3311,33 +3350,37 @@ static int fts3ColumnMethod(
  /* The column value supplied by SQLite must be in range. */
  assert( iCol>=0 && iCol<=p->nColumn+2 );

-  if( iCol==p->nColumn+1 ){
-    /* This call is a request for the "docid" column. Since "docid" is an 
-    ** alias for "rowid", use the xRowid() method to obtain the value.
-    */
-    sqlite3_result_int64(pCtx, pCsr->iPrevId);
-  }else if( iCol==p->nColumn ){
-    /* The extra column whose name is the same as the table.
-    ** Return a blob which is a pointer to the cursor.  */
-    sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
-  }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
-    sqlite3_result_int64(pCtx, pCsr->iLangid);
-  }else{
-    /* The requested column is either a user column (one that contains 
-    ** indexed data), or the language-id column.  */
-    rc = fts3CursorSeek(0, pCsr);
+  switch( iCol-p->nColumn ){
+    case 0:
+      /* The special 'table-name' column */
+      sqlite3_result_pointer(pCtx, pCsr);
+      break;

-    if( rc==SQLITE_OK ){
-      if( iCol==p->nColumn+2 ){
-        int iLangid = 0;
-        if( p->zLanguageid ){
-          iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1);
-        }
-        sqlite3_result_int(pCtx, iLangid);
-      }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
+    case 1:
+      /* The docid column */
+      sqlite3_result_int64(pCtx, pCsr->iPrevId);
+      break;
+
+    case 2:
+      if( pCsr->pExpr ){
+        sqlite3_result_int64(pCtx, pCsr->iLangid);
+        break;
+      }else if( p->zLanguageid==0 ){
+        sqlite3_result_int(pCtx, 0);
+        break;
+      }else{
+        iCol = p->nColumn;
+        /* fall-through */
+      }
+
+    default:
+      /* A user column. Or, if this is a full-table scan, possibly the
+      ** language-id column. Seek the cursor. */
+      rc = fts3CursorSeek(0, pCsr);
+      if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){
        sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
      }
-    }
+      break;
  }

  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
@@ -3386,8 +3429,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
  const u32 nMinMerge = 64;       /* Minimum amount of incr-merge work to do */

  Fts3Table *p = (Fts3Table*)pVtab;
-  int rc = sqlite3Fts3PendingTermsFlush(p);
+  int rc;
+  i64 iLastRowid = sqlite3_last_insert_rowid(p->db);

+  rc = sqlite3Fts3PendingTermsFlush(p);
  if( rc==SQLITE_OK 
   && p->nLeafAdd>(nMinMerge/16) 
   && p->nAutoincrmerge && p->nAutoincrmerge!=0xff
@@ -3402,6 +3447,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
    if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge);
  }
  sqlite3Fts3SegmentsClose(p);
+  sqlite3_set_last_insert_rowid(p->db, iLastRowid);
  return rc;
 }

@@ -3414,17 +3460,11 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
 static int fts3SetHasStat(Fts3Table *p){
  int rc = SQLITE_OK;
  if( p->bHasStat==2 ){
-    const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'";
-    char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName);
-    if( zSql ){
-      sqlite3_stmt *pStmt = 0;
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-      if( rc==SQLITE_OK ){
-        int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW);
-        rc = sqlite3_finalize(pStmt);
-        if( rc==SQLITE_OK ) p->bHasStat = bHasStat;
-      }
-      sqlite3_free(zSql);
+    char *zTbl = sqlite3_mprintf("%s_stat", p->zName);
+    if( zTbl ){
+      int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0);
+      sqlite3_free(zTbl);
+      p->bHasStat = (res==SQLITE_OK);
    }else{
      rc = SQLITE_NOMEM;
    }
@@ -3531,18 +3571,17 @@ static int fts3FunctionArg(
  sqlite3_value *pVal,            /* argv[0] passed to function */
  Fts3Cursor **ppCsr              /* OUT: Store cursor handle here */
 ){
-  Fts3Cursor *pRet;
-  if( sqlite3_value_type(pVal)!=SQLITE_BLOB 
-   || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
-  ){
+  int rc;
+  *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal);
+  if( (*ppCsr)!=0 ){
+    rc = SQLITE_OK;
+  }else{
    char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
    sqlite3_result_error(pContext, zErr, -1);
    sqlite3_free(zErr);
-    return SQLITE_ERROR;
+    rc = SQLITE_ERROR;
  }
-  memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
-  *ppCsr = pRet;
-  return SQLITE_OK;
+  return rc;
 }

 /*
@@ -3929,7 +3968,7 @@ int sqlite3Fts3Init(sqlite3 *db){
 #endif

  /* Create the virtual table wrapper around the hash-table and overload 
-  ** the two scalar functions. If this is successful, register the
+  ** the four scalar functions. If this is successful, register the
  ** module with sqlite.
  */
  if( SQLITE_OK==rc 
@@ -4512,7 +4551,7 @@ static int fts3EvalIncrPhraseNext(
  ** one incremental token. In which case the bIncr flag is set. */
  assert( p->bIncr==1 );

-  if( p->nToken==1 && p->bIncr ){
+  if( p->nToken==1 ){
    rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, 
        &pDL->iDocid, &pDL->pList, &pDL->nList
    );
@@ -4745,6 +4784,7 @@ static void fts3EvalTokenCosts(
 ** the number of overflow pages consumed by a record B bytes in size.
 */
 static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
+  int rc = SQLITE_OK;
  if( pCsr->nRowAvg==0 ){
    /* The average document size, which is required to calculate the cost
    ** of each doclist, has not yet been determined. Read the required 
@@ -4757,7 +4797,6 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
    ** data stored in all rows of each column of the table, from left
    ** to right.
    */
-    int rc;
    Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
    sqlite3_stmt *pStmt;
    sqlite3_int64 nDoc = 0;
@@ -4784,11 +4823,10 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
    pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
    assert( pCsr->nRowAvg>0 ); 
    rc = sqlite3_reset(pStmt);
-    if( rc!=SQLITE_OK ) return rc;
  }

  *pnPage = pCsr->nRowAvg;
-  return SQLITE_OK;
+  return rc;
 }

 /*
@@ -5138,7 +5176,8 @@ static void fts3EvalNextRow(
          pExpr->iDocid = pLeft->iDocid;
          pExpr->bEof = (pLeft->bEof || pRight->bEof);
          if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){
-            if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){
+            assert( pRight->eType==FTSQUERY_PHRASE );
+            if( pRight->pPhrase->doclist.aAll ){
              Fts3Doclist *pDl = &pRight->pPhrase->doclist;
              while( *pRc==SQLITE_OK && pRight->bEof==0 ){
                memset(pDl->pList, 0, pDl->nList);
@@ -5167,7 +5206,7 @@ static void fts3EvalNextRow(

        if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
          fts3EvalNextRow(pCsr, pLeft, pRc);
-        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
+        }else if( pLeft->bEof || iCmp>0 ){
          fts3EvalNextRow(pCsr, pRight, pRc);
        }else{
          fts3EvalNextRow(pCsr, pLeft, pRc);
@@ -5259,7 +5298,6 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
  */
  if( *pRc==SQLITE_OK 
   && pExpr->eType==FTSQUERY_NEAR 
-   && pExpr->bEof==0
   && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
  ){
    Fts3Expr *p; 
@@ -5268,42 +5306,39 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){

    /* Allocate temporary working space. */
    for(p=pExpr; p->pLeft; p=p->pLeft){
+      assert( p->pRight->pPhrase->doclist.nList>0 );
      nTmp += p->pRight->pPhrase->doclist.nList;
    }
    nTmp += p->pPhrase->doclist.nList;
-    if( nTmp==0 ){
+    aTmp = sqlite3_malloc(nTmp*2);
+    if( !aTmp ){
+      *pRc = SQLITE_NOMEM;
      res = 0;
    }else{
-      aTmp = sqlite3_malloc(nTmp*2);
-      if( !aTmp ){
-        *pRc = SQLITE_NOMEM;
-        res = 0;
-      }else{
-        char *aPoslist = p->pPhrase->doclist.pList;
-        int nToken = p->pPhrase->nToken;
+      char *aPoslist = p->pPhrase->doclist.pList;
+      int nToken = p->pPhrase->nToken;

-        for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
-          Fts3Phrase *pPhrase = p->pRight->pPhrase;
-          int nNear = p->nNear;
-          res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
-        }
-
-        aPoslist = pExpr->pRight->pPhrase->doclist.pList;
-        nToken = pExpr->pRight->pPhrase->nToken;
-        for(p=pExpr->pLeft; p && res; p=p->pLeft){
-          int nNear;
-          Fts3Phrase *pPhrase;
-          assert( p->pParent && p->pParent->pLeft==p );
-          nNear = p->pParent->nNear;
-          pPhrase = (
-              p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
-              );
-          res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
-        }
+      for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
+        Fts3Phrase *pPhrase = p->pRight->pPhrase;
+        int nNear = p->nNear;
+        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
      }

-      sqlite3_free(aTmp);
+      aPoslist = pExpr->pRight->pPhrase->doclist.pList;
+      nToken = pExpr->pRight->pPhrase->nToken;
+      for(p=pExpr->pLeft; p && res; p=p->pLeft){
+        int nNear;
+        Fts3Phrase *pPhrase;
+        assert( p->pParent && p->pParent->pLeft==p );
+        nNear = p->pParent->nNear;
+        pPhrase = (
+            p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
+        );
+        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
+      }
    }
+
+    sqlite3_free(aTmp);
  }

  return res;
@@ -18,6 +18,12 @@
 # define NDEBUG 1
 #endif

+/* FTS3/FTS4 require virtual tables */
+#ifdef SQLITE_OMIT_VIRTUALTABLE
+# undef SQLITE_ENABLE_FTS3
+# undef SQLITE_ENABLE_FTS4
+#endif
+
 /*
 ** FTS4 is really an extension for FTS3.  It is enabled using the
 ** SQLITE_ENABLE_FTS3 macro.  But to avoid confusion we also all
@@ -224,6 +230,7 @@ struct Fts3Table {
  ** statements is run and reset within a single virtual table API call. 
  */
  sqlite3_stmt *aStmt[40];
+  sqlite3_stmt *pSeekStmt;        /* Cache for fts3CursorSeekStmt() */

  char *zReadExprlist;
  char *zWriteExprlist;
@@ -293,6 +300,7 @@ struct Fts3Cursor {
  i16 eSearch;                    /* Search strategy (see below) */
  u8 isEof;                       /* True if at End Of Results */
  u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
+  u8 bSeekStmt;                   /* True if pStmt is a seek */
  sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
  Fts3Expr *pExpr;                /* Parsed MATCH query string */
  int iLangid;                    /* Language being queried for */
@@ -18,7 +18,14 @@
 ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
 */

-#include <tcl.h>
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 #include <string.h>
 #include <assert.h>

@@ -143,7 +150,7 @@ static int nm_match_count(
 /*
 ** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
 */
-static int fts3_near_match_cmd(
+static int SQLITE_TCLAPI fts3_near_match_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
@@ -278,7 +285,7 @@ static int fts3_near_match_cmd(
 **    # Restore initial incr-load settings:
 **    eval fts3_configure_incr_load $cfg
 */
-static int fts3_configure_incr_load_cmd(
+static int SQLITE_TCLAPI fts3_configure_incr_load_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
@@ -458,7 +465,7 @@ static int testTokenizerNext(
      if( pCsr->iLangid & 0x00000001 ){
        for(i=0; i<nToken; i++) pCsr->aBuffer[i] = pToken[i];
      }else{
-        for(i=0; i<nToken; i++) pCsr->aBuffer[i] = testTolower(pToken[i]);
+        for(i=0; i<nToken; i++) pCsr->aBuffer[i] = (char)testTolower(pToken[i]);
      }
      pCsr->iToken++;
      pCsr->iInput = (int)(p - pCsr->aInput);
@@ -488,7 +495,7 @@ static int testTokenizerLanguage(
 }
 #endif

-static int fts3_test_tokenizer_cmd(
+static int SQLITE_TCLAPI fts3_test_tokenizer_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
@@ -517,7 +524,7 @@ static int fts3_test_tokenizer_cmd(
  return TCL_OK;
 }

-static int fts3_test_varint_cmd(
+static int SQLITE_TCLAPI fts3_test_varint_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
@@ -526,7 +533,8 @@ static int fts3_test_varint_cmd(
 #ifdef SQLITE_ENABLE_FTS3
  char aBuf[24];
  int rc;
-  Tcl_WideInt w, w2;
+  Tcl_WideInt w;
+  sqlite3_int64 w2;
  int nByte, nByte2;

  if( objc!=2 ){
@@ -29,6 +29,18 @@
 #include <assert.h>
 #include <string.h>

+/*
+** Return true if the two-argument version of fts3_tokenizer()
+** has been activated via a prior call to sqlite3_db_config(db,
+** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
+*/
+static int fts3TokenizerEnabled(sqlite3_context *context){
+  sqlite3 *db = sqlite3_context_db_handle(context);
+  int isEnabled = 0;
+  sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled);
+  return isEnabled;
+}
+
 /*
 ** Implementation of the SQL scalar function for accessing the underlying 
 ** hash table. This function may be called as follows:
@@ -49,7 +61,7 @@
 ** is a blob containing the pointer stored as the hash data corresponding
 ** to string <key-name> (after the hash-table is updated, if applicable).
 */
-static void scalarFunc(
+static void fts3TokenizerFunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
@@ -67,27 +79,23 @@ static void scalarFunc(
  nName = sqlite3_value_bytes(argv[0])+1;

  if( argc==2 ){
-#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
-    void *pOld;
-    int n = sqlite3_value_bytes(argv[1]);
-    if( zName==0 || n!=sizeof(pPtr) ){
-      sqlite3_result_error(context, "argument type mismatch", -1);
+    if( fts3TokenizerEnabled(context) ){
+      void *pOld;
+      int n = sqlite3_value_bytes(argv[1]);
+      if( zName==0 || n!=sizeof(pPtr) ){
+        sqlite3_result_error(context, "argument type mismatch", -1);
+        return;
+      }
+      pPtr = *(void **)sqlite3_value_blob(argv[1]);
+      pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
+      if( pOld==pPtr ){
+        sqlite3_result_error(context, "out of memory", -1);
+      }
+    }else{
+      sqlite3_result_error(context, "fts3tokenize disabled", -1);
      return;
    }
-    pPtr = *(void **)sqlite3_value_blob(argv[1]);
-    pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
-    if( pOld==pPtr ){
-      sqlite3_result_error(context, "out of memory", -1);
-      return;
-    }
-#else
-    sqlite3_result_error(context, "fts3tokenize: " 
-        "disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER", -1
-    );
-    return;
-#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */
-  }else
-  {
+  }else{
    if( zName ){
      pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
    }
@@ -98,7 +106,6 @@ static void scalarFunc(
      return;
    }
  }
-
  sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
 }

@@ -217,7 +224,11 @@ int sqlite3Fts3InitTokenizer(

 #ifdef SQLITE_TEST

-#include <tcl.h>
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include <string.h>

 /*
@@ -336,7 +347,6 @@ finish:
  Tcl_DecrRefCount(pRet);
 }

-#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
 static
 int registerTokenizer(
  sqlite3 *db, 
@@ -358,7 +368,6 @@ int registerTokenizer(

  return sqlite3_finalize(pStmt);
 }
-#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */


 static
@@ -431,13 +440,13 @@ static void intTestFunc(
  assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );

  /* Test the storage function */
-#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
-  rc = registerTokenizer(db, "nosuchtokenizer", p1);
-  assert( rc==SQLITE_OK );
-  rc = queryTokenizer(db, "nosuchtokenizer", &p2);
-  assert( rc==SQLITE_OK );
-  assert( p2==p1 );
-#endif
+  if( fts3TokenizerEnabled(context) ){
+    rc = registerTokenizer(db, "nosuchtokenizer", p1);
+    assert( rc==SQLITE_OK );
+    rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+    assert( rc==SQLITE_OK );
+    assert( p2==p1 );
+  }

  sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
 }
@@ -453,7 +462,7 @@ static void intTestFunc(
 **    sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
 **
 ** This function adds a scalar function (see header comment above
-** scalarFunc() in this file for details) and, if ENABLE_TABLE is
+** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is
 ** defined at compilation time, a temporary virtual table (see header 
 ** comment above struct HashTableVtab) to the database schema. Both 
 ** provide read/write access to the contents of *pHash.
@@ -482,10 +491,10 @@ int sqlite3Fts3InitHashTable(
 #endif

  if( SQLITE_OK==rc ){
-    rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
+    rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0);
  }
  if( SQLITE_OK==rc ){
-    rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
+    rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0);
  }
 #ifdef SQLITE_TEST
  if( SQLITE_OK==rc ){
@@ -136,16 +136,16 @@ static int unicodeAddExceptions(
 ){
  const unsigned char *z = (const unsigned char *)zIn;
  const unsigned char *zTerm = &z[nIn];
-  int iCode;
+  unsigned int iCode;
  int nEntry = 0;

  assert( bAlnum==0 || bAlnum==1 );

  while( z<zTerm ){
    READ_UTF8(z, zTerm, iCode);
-    assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
-    if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 
-     && sqlite3FtsUnicodeIsdiacritic(iCode)==0 
+    assert( (sqlite3FtsUnicodeIsalnum((int)iCode) & 0xFFFFFFFE)==0 );
+    if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 
+     && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0 
    ){
      nEntry++;
    }
@@ -162,13 +162,13 @@ static int unicodeAddExceptions(
    z = (const unsigned char *)zIn;
    while( z<zTerm ){
      READ_UTF8(z, zTerm, iCode);
-      if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 
-       && sqlite3FtsUnicodeIsdiacritic(iCode)==0
+      if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 
+       && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
      ){
        int i, j;
-        for(i=0; i<nNew && aNew[i]<iCode; i++);
+        for(i=0; i<nNew && aNew[i]<(int)iCode; i++);
        for(j=nNew; j>i; j--) aNew[j] = aNew[j-1];
-        aNew[i] = iCode;
+        aNew[i] = (int)iCode;
        nNew++;
      }
    }
@@ -318,7 +318,7 @@ static int unicodeNext(
 ){
  unicode_cursor *pCsr = (unicode_cursor *)pC;
  unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer);
-  int iCode = 0;
+  unsigned int iCode = 0;
  char *zOut;
  const unsigned char *z = &pCsr->aInput[pCsr->iOff];
  const unsigned char *zStart = z;
@@ -330,7 +330,7 @@ static int unicodeNext(
  ** the input.  */
  while( z<zTerm ){
    READ_UTF8(z, zTerm, iCode);
-    if( unicodeIsAlnum(p, iCode) ) break;
+    if( unicodeIsAlnum(p, (int)iCode) ) break;
    zStart = z;
  }
  if( zStart>=zTerm ) return SQLITE_DONE;
@@ -350,7 +350,7 @@ static int unicodeNext(

    /* Write the folded case of the last character read to the output */
    zEnd = z;
-    iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic);
+    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
    if( iOut ){
      WRITE_UTF8(zOut, iOut);
    }
@@ -358,8 +358,8 @@ static int unicodeNext(
    /* If the cursor is not at EOF, read the next character */
    if( z>=zTerm ) break;
    READ_UTF8(z, zTerm, iCode);
-  }while( unicodeIsAlnum(p, iCode) 
-       || sqlite3FtsUnicodeIsdiacritic(iCode)
+  }while( unicodeIsAlnum(p, (int)iCode) 
+       || sqlite3FtsUnicodeIsdiacritic((int)iCode)
  );

  /* Set the output variables and return. */
@@ -127,9 +127,9 @@ int sqlite3FtsUnicodeIsalnum(int c){
    0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
  };

-  if( c<128 ){
-    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
-  }else if( c<(1<<22) ){
+  if( (unsigned int)c<128 ){
+    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
+  }else if( (unsigned int)c<(1<<22) ){
    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
    int iRes = 0;
    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
@@ -322,16 +322,17 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){

  int ret = c;

-  assert( c>=0 );
  assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );

  if( c<128 ){
    if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
  }else if( c<65536 ){
+    const struct TableEntry *p;
    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
    int iLo = 0;
    int iRes = -1;

+    assert( c>aEntry[0].iCode );
    while( iHi>=iLo ){
      int iTest = (iHi + iLo) / 2;
      int cmp = (c - aEntry[iTest].iCode);
@@ -342,14 +343,12 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
        iHi = iTest-1;
      }
    }
-    assert( iRes<0 || c>=aEntry[iRes].iCode );

-    if( iRes>=0 ){
-      const struct TableEntry *p = &aEntry[iRes];
-      if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
-        ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
-        assert( ret>0 );
-      }
+    assert( iRes>=0 && c>=aEntry[iRes].iCode );
+    p = &aEntry[iRes];
+    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
+      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
+      assert( ret>0 );
    }

    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
@@ -333,7 +333,8 @@ static int fts3SqlStmt(
 ** of the oldest level in the db that contains at least ? segments. Or,
 ** if no level in the FTS index contains more than ? segments, the statement
 ** returns zero rows.  */
-/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
+/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' "
+         "  GROUP BY level HAVING cnt>=?"
         "  ORDER BY (level %% 1024) ASC LIMIT 1",

 /* Estimate the upper limit on the number of leaf nodes in a new segment
@@ -406,7 +407,8 @@ static int fts3SqlStmt(
    if( !zSql ){
      rc = SQLITE_NOMEM;
    }else{
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
+      rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
+                              &pStmt, NULL);
      sqlite3_free(zSql);
      assert( rc==SQLITE_OK || pStmt==0 );
      p->aStmt[eStmt] = pStmt;
@@ -3194,7 +3196,7 @@ static int fts3SegmentMerge(
    ** segment. The level of the new segment is equal to the numerically
    ** greatest segment level currently present in the database for this
    ** index. The idx of the new segment is always 0.  */
-    if( csr.nSegment==1 ){
+    if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){
      rc = SQLITE_DONE;
      goto finished;
    }
@@ -4836,10 +4838,11 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
    ** set nSeg to -1.
    */
    rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
-    sqlite3_bind_int(pFindLevel, 1, nMin);
+    sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin));
    if( sqlite3_step(pFindLevel)==SQLITE_ROW ){
      iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
-      nSeg = nMin;
+      nSeg = sqlite3_column_int(pFindLevel, 1);
+      assert( nSeg>=2 );
    }else{
      nSeg = -1;
    }
@@ -4954,11 +4957,14 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
 ** Convert the text beginning at *pz into an integer and return
 ** its value.  Advance *pz to point to the first character past
 ** the integer.
+**
+** This function used for parameters to merge= and incrmerge=
+** commands. 
 */
 static int fts3Getint(const char **pz){
  const char *z = *pz;
  int i = 0;
-  while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0';
+  while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0';
  *pz = z;
  return i;
 }
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+srcdir=`dirname $(dirname $(dirname $(dirname $0)))`
+./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt
+
+echo ""
+
+for f in `ls $srcdir/ext/fts3/*.c` 
+do
+  f=`basename $f`
+  echo -ne "$f: "
+  gcov -b $f | grep Taken | sed 's/Taken at least once://'
+done
+
@@ -398,8 +398,8 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){
    if( sqlite3_step(pStmt)==SQLITE_ROW
     && (nLeaf = sqlite3_column_int(pStmt, 0))>0
    ){
-      nIdx = sqlite3_column_int(pStmt, 5);
      sqlite3_int64 sz;
+      nIdx = sqlite3_column_int(pStmt, 5);
      printf("For level %d:\n", i);
      printf("  Number of indexes...................... %9d\n", nIdx);
      printf("  Number of leaf segments................ %9d\n", nLeaf);
@@ -226,9 +226,9 @@ proc print_isalnum {zFunc lRange} {
  an_print_range_array $lRange
  an_print_ascii_bitmap $lRange
  puts {
-  if( c<128 ){
-    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
-  }else if( c<(1<<22) ){
+  if( (unsigned int)c<128 ){
+    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
+  }else if( (unsigned int)c<(1<<22) ){
    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
    int iRes = 0;
    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
@@ -143,11 +143,13 @@ struct Fts5PhraseIter {
 **       ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
 **
 **   with $p set to a phrase equivalent to the phrase iPhrase of the
-**   current query is executed. For each row visited, the callback function
-**   passed as the fourth argument is invoked. The context and API objects 
-**   passed to the callback function may be used to access the properties of
-**   each matched row. Invoking Api.xUserData() returns a copy of the pointer
-**   passed as the third argument to pUserData.
+**   current query is executed. Any column filter that applies to
+**   phrase iPhrase of the current query is included in $p. For each 
+**   row visited, the callback function passed as the fourth argument 
+**   is invoked. The context and API objects passed to the callback 
+**   function may be used to access the properties of each matched row.
+**   Invoking Api.xUserData() returns a copy of the pointer passed as 
+**   the third argument to pUserData.
 **
 **   If the callback function returns any value other than SQLITE_OK, the
 **   query is abandoned and the xQueryPhrase function returns immediately.
@@ -316,7 +318,7 @@ struct Fts5ExtensionApi {
 ** behaviour. The structure methods are expected to function as follows:
 **
 ** xCreate:
-**   This function is used to allocate and inititalize a tokenizer instance.
+**   This function is used to allocate and initialize a tokenizer instance.
 **   A tokenizer instance is required to actually tokenize text.
 **
 **   The first argument passed to this function is a copy of the (void*)
@@ -575,4 +577,3 @@ struct fts5_api {
 #endif

 #endif /* _FTS5_H */
-
@@ -26,10 +26,13 @@ SQLITE_EXTENSION_INIT1
 typedef unsigned char  u8;
 typedef unsigned int   u32;
 typedef unsigned short u16;
+typedef short i16;
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;

-#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
+#ifndef ArraySize
+# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
+#endif

 #define testcase(x)
 #define ALWAYS(x) 1
@@ -46,6 +49,10 @@ typedef sqlite3_uint64 u64;

 #endif

+/* Truncate very long tokens to this many bytes. Hard limit is 
+** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
+** field that occurs at the start of each leaf page (see fts5_index.c). */
+#define FTS5_MAX_TOKEN_SIZE 32768

 /*
 ** Maximum number of prefix indexes on single FTS5 table. This must be
@@ -80,6 +87,16 @@ extern int sqlite3_fts5_may_be_corrupt;
 # define assert_nc(x) assert(x)
 #endif

+/* Mark a function parameter as unused, to suppress nuisance compiler
+** warnings. */
+#ifndef UNUSED_PARAM
+# define UNUSED_PARAM(X)  (void)(X)
+#endif
+
+#ifndef UNUSED_PARAM2
+# define UNUSED_PARAM2(X, Y)  (void)(X), (void)(Y)
+#endif
+
 typedef struct Fts5Global Fts5Global;
 typedef struct Fts5Colset Fts5Colset;

@@ -161,6 +178,7 @@ struct Fts5Config {
  int pgsz;                       /* Approximate page size used in %_data */
  int nAutomerge;                 /* 'automerge' setting */
  int nCrisisMerge;               /* Maximum allowed segments per level */
+  int nUsermerge;                 /* 'usermerge' setting */
  int nHashSize;                  /* Bytes of memory for in-memory hash */
  char *zRank;                    /* Name of rank function */
  char *zRankArgs;                /* Arguments to rank function */
@@ -345,16 +363,6 @@ struct Fts5IndexIter {
 int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
 int sqlite3Fts5IndexClose(Fts5Index *p);

-/*
-** for(
-**   sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter);
-**   0==sqlite3Fts5IterEof(pIter);
-**   sqlite3Fts5IterNext(pIter)
-** ){
-**   i64 iRowid = sqlite3Fts5IterRowid(pIter);
-** }
-*/
-
 /*
 ** Return a simple checksum value based on the arguments.
 */
@@ -396,7 +404,6 @@ int sqlite3Fts5IndexQuery(
 */
 int sqlite3Fts5IterNext(Fts5IndexIter*);
 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
-i64 sqlite3Fts5IterRowid(Fts5IndexIter*);

 /*
 ** Close an iterator opened by sqlite3Fts5IndexQuery().
@@ -439,9 +446,9 @@ int sqlite3Fts5IndexBeginWrite(

 /*
 ** Flush any data stored in the in-memory hash tables to the database.
-** If the bCommit flag is true, also close any open blob handles.
+** Also close any open blob handles.
 */
-int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit);
+int sqlite3Fts5IndexSync(Fts5Index *p);

 /*
 ** Discard any data stored in the in-memory hash tables. Do not write it
@@ -479,6 +486,7 @@ int sqlite3Fts5IndexReads(Fts5Index *p);
 int sqlite3Fts5IndexReinit(Fts5Index *p);
 int sqlite3Fts5IndexOptimize(Fts5Index *p);
 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
+int sqlite3Fts5IndexReset(Fts5Index *p);

 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);

@@ -610,7 +618,7 @@ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);

-int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
+int sqlite3Fts5StorageSync(Fts5Storage *p);
 int sqlite3Fts5StorageRollback(Fts5Storage *p);

 int sqlite3Fts5StorageConfigValue(
@@ -621,6 +629,7 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
+int sqlite3Fts5StorageReset(Fts5Storage *p);

 /*
 ** End of interface to code in fts5_storage.c.
@@ -645,6 +654,7 @@ struct Fts5Token {
 /* Parse a MATCH expression. */
 int sqlite3Fts5ExprNew(
  Fts5Config *pConfig, 
+  int iCol,                       /* Column on LHS of MATCH operator */
  const char *zExpr,
  Fts5Expr **ppNew, 
  char **pzErr
@@ -679,9 +689,8 @@ int sqlite3Fts5ExprPopulatePoslists(
    Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 );
 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
-void sqlite3Fts5ExprClearEof(Fts5Expr*);

-int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
+int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);

 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);

@@ -700,6 +709,12 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
  Fts5ExprNearset *pNear
 );

+Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
+  Fts5Parse *pParse,
+  Fts5ExprNode *pLeft,
+  Fts5ExprNode *pRight
+);
+
 Fts5ExprPhrase *sqlite3Fts5ParseTerm(
  Fts5Parse *pParse, 
  Fts5ExprPhrase *pPhrase, 
@@ -724,7 +739,8 @@ void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
-void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
+void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
+Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);

@@ -158,6 +158,8 @@ static int fts5HighlightCb(
  int rc = SQLITE_OK;
  int iPos;

+  UNUSED_PARAM2(pToken, nToken);
+
  if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
  iPos = p->iPos++;

@@ -187,7 +189,7 @@ static int fts5HighlightCb(
  if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
    p->iOff = iEndOff;
-    if( iPos<p->iter.iEnd ){
+    if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
      fts5HighlightAppend(&rc, p, p->zClose, -1);
    }
  }
@@ -244,6 +246,118 @@ static void fts5HighlightFunction(
 ** End of highlight() implementation.
 **************************************************************************/

+/*
+** Context object passed to the fts5SentenceFinderCb() function.
+*/
+typedef struct Fts5SFinder Fts5SFinder;
+struct Fts5SFinder {
+  int iPos;                       /* Current token position */
+  int nFirstAlloc;                /* Allocated size of aFirst[] */
+  int nFirst;                     /* Number of entries in aFirst[] */
+  int *aFirst;                    /* Array of first token in each sentence */
+  const char *zDoc;               /* Document being tokenized */
+};
+
+/*
+** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
+** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
+** error occurs.
+*/
+static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
+  if( p->nFirstAlloc==p->nFirst ){
+    int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
+    int *aNew;
+
+    aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int));
+    if( aNew==0 ) return SQLITE_NOMEM;
+    p->aFirst = aNew;
+    p->nFirstAlloc = nNew;
+  }
+  p->aFirst[p->nFirst++] = iAdd;
+  return SQLITE_OK;
+}
+
+/*
+** This function is an xTokenize() callback used by the auxiliary snippet()
+** function. Its job is to identify tokens that are the first in a sentence.
+** For each such token, an entry is added to the SFinder.aFirst[] array.
+*/
+static int fts5SentenceFinderCb(
+  void *pContext,                 /* Pointer to HighlightContext object */
+  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStartOff,                  /* Start offset of token */
+  int iEndOff                     /* End offset of token */
+){
+  int rc = SQLITE_OK;
+
+  UNUSED_PARAM2(pToken, nToken);
+  UNUSED_PARAM(iEndOff);
+
+  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
+    Fts5SFinder *p = (Fts5SFinder*)pContext;
+    if( p->iPos>0 ){
+      int i;
+      char c = 0;
+      for(i=iStartOff-1; i>=0; i--){
+        c = p->zDoc[i];
+        if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
+      }
+      if( i!=iStartOff-1 && (c=='.' || c==':') ){
+        rc = fts5SentenceFinderAdd(p, p->iPos);
+      }
+    }else{
+      rc = fts5SentenceFinderAdd(p, 0);
+    }
+    p->iPos++;
+  }
+  return rc;
+}
+
+static int fts5SnippetScore(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  int nDocsize,                   /* Size of column in tokens */
+  unsigned char *aSeen,           /* Array with one element per query phrase */
+  int iCol,                       /* Column to score */
+  int iPos,                       /* Starting offset to score */
+  int nToken,                     /* Max tokens per snippet */
+  int *pnScore,                   /* OUT: Score */
+  int *piPos                      /* OUT: Adjusted offset */
+){
+  int rc;
+  int i;
+  int ip = 0;
+  int ic = 0;
+  int iOff = 0;
+  int iFirst = -1;
+  int nInst;
+  int nScore = 0;
+  int iLast = 0;
+
+  rc = pApi->xInstCount(pFts, &nInst);
+  for(i=0; i<nInst && rc==SQLITE_OK; i++){
+    rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
+    if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){
+      nScore += (aSeen[ip] ? 1 : 1000);
+      aSeen[ip] = 1;
+      if( iFirst<0 ) iFirst = iOff;
+      iLast = iOff + pApi->xPhraseSize(pFts, ip);
+    }
+  }
+
+  *pnScore = nScore;
+  if( piPos ){
+    int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
+    if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
+    if( iAdj<0 ) iAdj = 0;
+    *piPos = iAdj;
+  }
+
+  return rc;
+}
+
 /*
 ** Implementation of snippet() function.
 */
@@ -265,9 +379,10 @@ static void fts5SnippetFunction(
  unsigned char *aSeen;           /* Array of "seen instance" flags */
  int iBestCol;                   /* Column containing best snippet */
  int iBestStart = 0;             /* First token of best snippet */
-  int iBestLast;                  /* Last token of best snippet */
  int nBestScore = 0;             /* Score of best snippet */
  int nColSize = 0;               /* Total size of iBestCol in tokens */
+  Fts5SFinder sFinder;            /* Used to find the beginnings of sentences */
+  int nCol;

  if( nVal!=5 ){
    const char *zErr = "wrong number of arguments to function snippet()";
@@ -275,13 +390,13 @@ static void fts5SnippetFunction(
    return;
  }

+  nCol = pApi->xColumnCount(pFts);
  memset(&ctx, 0, sizeof(HighlightContext));
  iCol = sqlite3_value_int(apVal[0]);
  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
  zEllips = (const char*)sqlite3_value_text(apVal[3]);
  nToken = sqlite3_value_int(apVal[4]);
-  iBestLast = nToken-1;

  iBestCol = (iCol>=0 ? iCol : 0);
  nPhrase = pApi->xPhraseCount(pFts);
@@ -289,65 +404,94 @@ static void fts5SnippetFunction(
  if( aSeen==0 ){
    rc = SQLITE_NOMEM;
  }
-
  if( rc==SQLITE_OK ){
    rc = pApi->xInstCount(pFts, &nInst);
  }
-  for(i=0; rc==SQLITE_OK && i<nInst; i++){
-    int ip, iSnippetCol, iStart;
-    memset(aSeen, 0, nPhrase);
-    rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart);
-    if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){
-      int nScore = 1000;
-      int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip);
-      int j;
-      aSeen[ip] = 1;

-      for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
-        int ic; int io; int iFinal;
-        rc = pApi->xInst(pFts, j, &ip, &ic, &io);
-        iFinal = io + pApi->xPhraseSize(pFts, ip) - 1;
-        if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){
-          nScore += aSeen[ip] ? 1000 : 1;
-          aSeen[ip] = 1;
-          if( iFinal>iLast ) iLast = iFinal;
+  memset(&sFinder, 0, sizeof(Fts5SFinder));
+  for(i=0; i<nCol; i++){
+    if( iCol<0 || iCol==i ){
+      int nDoc;
+      int nDocsize;
+      int ii;
+      sFinder.iPos = 0;
+      sFinder.nFirst = 0;
+      rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xTokenize(pFts, 
+          sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
+      );
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xColumnSize(pFts, i, &nDocsize);
+      if( rc!=SQLITE_OK ) break;
+
+      for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
+        int ip, ic, io;
+        int iAdj;
+        int nScore;
+        int jj;
+
+        rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
+        if( ic!=i || rc!=SQLITE_OK ) continue;
+        memset(aSeen, 0, nPhrase);
+        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
+            io, nToken, &nScore, &iAdj
+        );
+        if( rc==SQLITE_OK && nScore>nBestScore ){
+          nBestScore = nScore;
+          iBestCol = i;
+          iBestStart = iAdj;
+          nColSize = nDocsize;
        }
-      }

-      if( rc==SQLITE_OK && nScore>nBestScore ){
-        iBestCol = iSnippetCol;
-        iBestStart = iStart;
-        iBestLast = iLast;
-        nBestScore = nScore;
+        if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){
+          for(jj=0; jj<(sFinder.nFirst-1); jj++){
+            if( sFinder.aFirst[jj+1]>io ) break;
+          }
+
+          if( sFinder.aFirst[jj]<io ){
+            memset(aSeen, 0, nPhrase);
+            rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
+              sFinder.aFirst[jj], nToken, &nScore, 0
+            );
+
+            nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
+            if( rc==SQLITE_OK && nScore>nBestScore ){
+              nBestScore = nScore;
+              iBestCol = i;
+              iBestStart = sFinder.aFirst[jj];
+              nColSize = nDocsize;
+            }
+          }
+        }
      }
    }
  }

-  if( rc==SQLITE_OK ){
-    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
-  }
  if( rc==SQLITE_OK ){
    rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
  }
+  if( rc==SQLITE_OK && nColSize==0 ){
+    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
+  }
  if( ctx.zIn ){
    if( rc==SQLITE_OK ){
      rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
    }

-    if( (iBestStart+nToken-1)>iBestLast ){
-      iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
-    }
-    if( iBestStart+nToken>nColSize ){
-      iBestStart = nColSize - nToken;
-    }
-    if( iBestStart<0 ) iBestStart = 0;
-
    ctx.iRangeStart = iBestStart;
    ctx.iRangeEnd = iBestStart + nToken - 1;

    if( iBestStart>0 ){
      fts5HighlightAppend(&rc, &ctx, zEllips, -1);
    }
+
+    /* Advance iterator ctx.iter so that it points to the first coalesced
+    ** phrase instance at or following position iBestStart. */
+    while( ctx.iter.iStart>=0 && ctx.iter.iStart<iBestStart && rc==SQLITE_OK ){
+      rc = fts5CInstIterNext(&ctx.iter);
+    }
+
    if( rc==SQLITE_OK ){
      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
    }
@@ -356,15 +500,15 @@ static void fts5SnippetFunction(
    }else{
      fts5HighlightAppend(&rc, &ctx, zEllips, -1);
    }
-
-    if( rc==SQLITE_OK ){
-      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
-    }else{
-      sqlite3_result_error_code(pCtx, rc);
-    }
-    sqlite3_free(ctx.zOut);
  }
+  if( rc==SQLITE_OK ){
+    sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_error_code(pCtx, rc);
+  }
+  sqlite3_free(ctx.zOut);
  sqlite3_free(aSeen);
+  sqlite3_free(sFinder.aFirst);
 }

 /************************************************************************/
@@ -391,6 +535,7 @@ static int fts5CountCb(
  void *pUserData                 /* Pointer to sqlite3_int64 variable */
 ){
  sqlite3_int64 *pn = (sqlite3_int64*)pUserData;
+  UNUSED_PARAM2(pApi, pFts);
  (*pn)++;
  return SQLITE_OK;
 }
@@ -67,9 +67,11 @@ void sqlite3Fts5BufferAppendBlob(
  const u8 *pData
 ){
  assert_nc( *pRc || nData>=0 );
-  if( fts5BufferGrow(pRc, pBuf, nData) ) return;
-  memcpy(&pBuf->p[pBuf->n], pData, nData);
-  pBuf->n += nData;
+  if( nData ){
+    if( fts5BufferGrow(pRc, pBuf, nData) ) return;
+    memcpy(&pBuf->p[pBuf->n], pData, nData);
+    pBuf->n += nData;
+  }
 }

 /*
@@ -236,7 +238,7 @@ int sqlite3Fts5PoslistWriterAppend(
  Fts5PoslistWriter *pWriter,
  i64 iPos
 ){
-  int rc;
+  int rc = 0;   /* Initialized only to suppress erroneous warning from Clang */
  if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
  sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
  return SQLITE_OK;
@@ -246,8 +248,8 @@ void *sqlite3Fts5MallocZero(int *pRc, int nByte){
  void *pRet = 0;
  if( *pRc==SQLITE_OK ){
    pRet = sqlite3_malloc(nByte);
-    if( pRet==0 && nByte>0 ){
-      *pRc = SQLITE_NOMEM;
+    if( pRet==0 ){
+      if( nByte>0 ) *pRc = SQLITE_NOMEM;
    }else{
      memset(pRet, 0, nByte);
    }
@@ -390,6 +392,3 @@ void sqlite3Fts5TermsetFree(Fts5Termset *p){
    sqlite3_free(p);
  }
 }
-
-
-
@@ -18,6 +18,7 @@

 #define FTS5_DEFAULT_PAGE_SIZE   4050
 #define FTS5_DEFAULT_AUTOMERGE      4
+#define FTS5_DEFAULT_USERMERGE      4
 #define FTS5_DEFAULT_CRISISMERGE   16
 #define FTS5_DEFAULT_HASHSIZE    (1024*1024)

@@ -206,7 +207,7 @@ static int fts5ConfigSetEnum(
  const char *zEnum, 
  int *peVal
 ){
-  int nEnum = strlen(zEnum);
+  int nEnum = (int)strlen(zEnum);
  int i;
  int iVal = -1;

@@ -441,7 +442,9 @@ static const char *fts5ConfigGobbleWord(
      *pbQuoted = 1;
    }else{
      zRet = fts5ConfigSkipBareword(zIn);
-      zOut[zRet-zIn] = '\0';
+      if( zRet ){
+        zOut[zRet-zIn] = '\0';
+      }
    }
  }

@@ -857,6 +860,18 @@ int sqlite3Fts5ConfigSetValue(
    }
  }

+  else if( 0==sqlite3_stricmp(zKey, "usermerge") ){
+    int nUsermerge = -1;
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      nUsermerge = sqlite3_value_int(pVal);
+    }
+    if( nUsermerge<2 || nUsermerge>16 ){
+      *pbBadkey = 1;
+    }else{
+      pConfig->nUsermerge = nUsermerge;
+    }
+  }
+
  else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
    int nCrisisMerge = -1;
    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
@@ -903,6 +918,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
  /* Set default values */
  pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
  pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
+  pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
  pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
  pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;

@@ -943,4 +959,3 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
  }
  return rc;
 }
-
@@ -167,6 +167,7 @@ static int fts5ExprGetToken(
    case ',':  tok = FTS5_COMMA; break;
    case '+':  tok = FTS5_PLUS;  break;
    case '*':  tok = FTS5_STAR;  break;
+    case '-':  tok = FTS5_MINUS; break;
    case '\0': tok = FTS5_EOF;   break;

    case '"': {
@@ -212,6 +213,7 @@ static void fts5ParseFree(void *p){ sqlite3_free(p); }

 int sqlite3Fts5ExprNew(
  Fts5Config *pConfig,            /* FTS5 Configuration */
+  int iCol,
  const char *zExpr,              /* Expression text */
  Fts5Expr **ppNew, 
  char **pzErr
@@ -236,6 +238,18 @@ int sqlite3Fts5ExprNew(
  }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
  sqlite3Fts5ParserFree(pEngine, fts5ParseFree);

+  /* If the LHS of the MATCH expression was a user column, apply the
+  ** implicit column-filter.  */
+  if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
+    int n = sizeof(Fts5Colset);
+    Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
+    if( pColset ){
+      pColset->nCol = 1;
+      pColset->aiCol[0] = iCol;
+      sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset);
+    }
+  }
+
  assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
  if( sParse.rc==SQLITE_OK ){
    *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
@@ -258,6 +272,8 @@ int sqlite3Fts5ExprNew(
      pNew->nPhrase = sParse.nPhrase;
      sParse.apPhrase = 0;
    }
+  }else{
+    sqlite3Fts5ParseNodeFree(sParse.pExpr);
  }

  sqlite3_free(sParse.apPhrase);
@@ -320,8 +336,6 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
 */
 static int fts5ExprSynonymList(
  Fts5ExprTerm *pTerm, 
-  int bCollist, 
-  Fts5Colset *pColset,
  i64 iRowid,
  Fts5Buffer *pBuf,               /* Use this buffer for space if required */
  u8 **pa, int *pn
@@ -405,7 +419,6 @@ static int fts5ExprSynonymList(
 */
 static int fts5ExprPhraseIsMatch(
  Fts5ExprNode *pNode,            /* Node pPhrase belongs to */
-  Fts5Colset *pColset,            /* Restrict matches to these columns */
  Fts5ExprPhrase *pPhrase,        /* Phrase object to initialize */
  int *pbMatch                    /* OUT: Set to true if really a match */
 ){
@@ -434,9 +447,7 @@ static int fts5ExprPhraseIsMatch(
    u8 *a = 0;
    if( pTerm->pSynonym ){
      Fts5Buffer buf = {0, 0, 0};
-      rc = fts5ExprSynonymList(
-          pTerm, 0, pColset, pNode->iRowid, &buf, &a, &n
-      );
+      rc = fts5ExprSynonymList(pTerm, pNode->iRowid, &buf, &a, &n);
      if( rc ){
        sqlite3_free(a);
        goto ismatch_out;
@@ -727,7 +738,7 @@ static int fts5ExprNearTest(
      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
        int bMatch = 0;
-        rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
+        rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
        if( bMatch==0 ) break;
      }else{
        Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
@@ -748,49 +759,61 @@ static int fts5ExprNearTest(
 ** Initialize all term iterators in the pNear object. If any term is found
 ** to match no documents at all, return immediately without initializing any
 ** further iterators.
+**
+** If an error occurs, return an SQLite error code. Otherwise, return
+** SQLITE_OK. It is not considered an error if some term matches zero
+** documents.
 */
 static int fts5ExprNearInitAll(
  Fts5Expr *pExpr,
  Fts5ExprNode *pNode
 ){
  Fts5ExprNearset *pNear = pNode->pNear;
-  int i, j;
-  int rc = SQLITE_OK;
+  int i;

  assert( pNode->bNomatch==0 );
-  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
+  for(i=0; i<pNear->nPhrase; i++){
    Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
-    for(j=0; j<pPhrase->nTerm; j++){
-      Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
-      Fts5ExprTerm *p;
-      int bEof = 1;
+    if( pPhrase->nTerm==0 ){
+      pNode->bEof = 1;
+      return SQLITE_OK;
+    }else{
+      int j;
+      for(j=0; j<pPhrase->nTerm; j++){
+        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
+        Fts5ExprTerm *p;
+        int bHit = 0;

-      for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){
-        if( p->pIter ){
-          sqlite3Fts5IterClose(p->pIter);
-          p->pIter = 0;
+        for(p=pTerm; p; p=p->pSynonym){
+          int rc;
+          if( p->pIter ){
+            sqlite3Fts5IterClose(p->pIter);
+            p->pIter = 0;
+          }
+          rc = sqlite3Fts5IndexQuery(
+              pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
+              (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
+              (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
+              pNear->pColset,
+              &p->pIter
+          );
+          assert( (rc==SQLITE_OK)==(p->pIter!=0) );
+          if( rc!=SQLITE_OK ) return rc;
+          if( 0==sqlite3Fts5IterEof(p->pIter) ){
+            bHit = 1;
+          }
        }
-        rc = sqlite3Fts5IndexQuery(
-            pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
-            (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
-            (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
-            pNear->pColset,
-            &p->pIter
-        );
-        assert( rc==SQLITE_OK || p->pIter==0 );
-        if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
-          bEof = 0;
-        }
-      }

-      if( bEof ){
-        pNode->bEof = 1;
-        return rc;
+        if( bHit==0 ){
+          pNode->bEof = 1;
+          return SQLITE_OK;
+        }
      }
    }
  }

-  return rc;
+  pNode->bEof = 0;
+  return SQLITE_OK;
 }

 /*
@@ -923,7 +946,7 @@ static int fts5ExprNodeTest_STRING(
          }
        }else{
          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
-          if( pIter->iRowid==iLast ) continue;
+          if( pIter->iRowid==iLast || pIter->bEof ) continue;
          bMatch = 0;
          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
            return rc;
@@ -1100,7 +1123,10 @@ static int fts5ExprNodeNext_OR(
       || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
      ){
        int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
-        if( rc!=SQLITE_OK ) return rc;
+        if( rc!=SQLITE_OK ){
+          pNode->bNomatch = 0;
+          return rc;
+        }
      }
    }
  }
@@ -1131,7 +1157,10 @@ static int fts5ExprNodeTest_AND(
      if( cmp>0 ){
        /* Advance pChild until it points to iLast or laster */
        rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
-        if( rc!=SQLITE_OK ) return rc;
+        if( rc!=SQLITE_OK ){
+          pAnd->bNomatch = 0;
+          return rc;
+        }
      }

      /* If the child node is now at EOF, so is the parent AND node. Otherwise,
@@ -1170,6 +1199,8 @@ static int fts5ExprNodeNext_AND(
  int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
  if( rc==SQLITE_OK ){
    rc = fts5ExprNodeTest_AND(pExpr, pNode);
+  }else{
+    pNode->bNomatch = 0;
  }
  return rc;
 }
@@ -1212,6 +1243,9 @@ static int fts5ExprNodeNext_NOT(
  if( rc==SQLITE_OK ){
    rc = fts5ExprNodeTest_NOT(pExpr, pNode);
  }
+  if( rc!=SQLITE_OK ){
+    pNode->bNomatch = 0;
+  }
  return rc;
 }

@@ -1273,6 +1307,8 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
  if( Fts5NodeIsString(pNode) ){
    /* Initialize all term iterators in the NEAR object. */
    rc = fts5ExprNearInitAll(pExpr, pNode);
+  }else if( pNode->xNext==0 ){
+    pNode->bEof = 1;
  }else{
    int i;
    int nEof = 0;
@@ -1324,23 +1360,25 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
 */
 int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){
  Fts5ExprNode *pRoot = p->pRoot;
-  int rc = SQLITE_OK;
-  if( pRoot->xNext ){
-    p->pIndex = pIdx;
-    p->bDesc = bDesc;
-    rc = fts5ExprNodeFirst(p, pRoot);
+  int rc;                         /* Return code */

-    /* If not at EOF but the current rowid occurs earlier than iFirst in
-    ** the iteration order, move to document iFirst or later. */
-    if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
-      rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
-    }
+  p->pIndex = pIdx;
+  p->bDesc = bDesc;
+  rc = fts5ExprNodeFirst(p, pRoot);

-    /* If the iterator is not at a real match, skip forward until it is. */
-    while( pRoot->bNomatch ){
-      assert( pRoot->bEof==0 && rc==SQLITE_OK );
-      rc = fts5ExprNodeNext(p, pRoot, 0, 0);
-    }
+  /* If not at EOF but the current rowid occurs earlier than iFirst in
+  ** the iteration order, move to document iFirst or later. */
+  if( rc==SQLITE_OK 
+   && 0==pRoot->bEof 
+   && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 
+  ){
+    rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
+  }
+
+  /* If the iterator is not at a real match, skip forward until it is. */
+  while( pRoot->bNomatch ){
+    assert( pRoot->bEof==0 && rc==SQLITE_OK );
+    rc = fts5ExprNodeNext(p, pRoot, 0, 0);
  }
  return rc;
 }
@@ -1449,6 +1487,21 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
    sqlite3Fts5ParseNearsetFree(pNear);
    sqlite3Fts5ParsePhraseFree(pPhrase);
  }else{
+    if( pRet->nPhrase>0 ){
+      Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1];
+      assert( pLast==pParse->apPhrase[pParse->nPhrase-2] );
+      if( pPhrase->nTerm==0 ){
+        fts5ExprPhraseFree(pPhrase);
+        pRet->nPhrase--;
+        pParse->nPhrase--;
+        pPhrase = pLast;
+      }else if( pLast->nTerm==0 ){
+        fts5ExprPhraseFree(pLast);
+        pParse->apPhrase[pParse->nPhrase-2] = pPhrase;
+        pParse->nPhrase--;
+        pRet->nPhrase--;
+      }
+    }
    pRet->apPhrase[pRet->nPhrase++] = pPhrase;
  }
  return pRet;
@@ -1476,11 +1529,13 @@ static int fts5ParseTokenize(
  TokenCtx *pCtx = (TokenCtx*)pContext;
  Fts5ExprPhrase *pPhrase = pCtx->pPhrase;

+  UNUSED_PARAM2(iUnused1, iUnused2);
+
  /* If an error has already occurred, this is a no-op */
  if( pCtx->rc!=SQLITE_OK ) return pCtx->rc;
+  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;

-  assert( pPhrase==0 || pPhrase->nTerm>0 );
-  if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){
+  if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){
    Fts5ExprTerm *pSyn;
    int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
    pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
@@ -1570,7 +1625,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(

  rc = fts5ParseStringFromToken(pToken, &z);
  if( rc==SQLITE_OK ){
-    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
+    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
    int n;
    sqlite3Fts5Dequote(z);
    n = (int)strlen(z);
@@ -1581,7 +1636,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
    pParse->rc = rc;
    fts5ExprPhraseFree(sCtx.pPhrase);
    sCtx.pPhrase = 0;
-  }else if( sCtx.pPhrase ){
+  }else{

    if( pAppend==0 ){
      if( (pParse->nPhrase % 8)==0 ){
@@ -1598,9 +1653,14 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
      pParse->nPhrase++;
    }

+    if( sCtx.pPhrase==0 ){
+      /* This happens when parsing a token or quoted phrase that contains
+      ** no token characters at all. (e.g ... MATCH '""'). */
+      sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase));
+    }else if( sCtx.pPhrase->nTerm ){
+      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
+    }
    pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
-    assert( sCtx.pPhrase->nTerm>0 );
-    sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
  }

  return sCtx.pPhrase;
@@ -1611,22 +1671,16 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 ** expression passed as the second argument.
 */
 int sqlite3Fts5ExprClonePhrase(
-  Fts5Config *pConfig,
  Fts5Expr *pExpr, 
  int iPhrase, 
  Fts5Expr **ppNew
 ){
  int rc = SQLITE_OK;             /* Return code */
  Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
-  int i;                          /* Used to iterate through phrase terms */
-
  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
-
  TokenCtx sCtx = {0,0};          /* Context object for fts5ParseTokenize */

-
  pOrig = pExpr->apExprPhrase[iPhrase];
-
  pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
  if( rc==SQLITE_OK ){
    pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, 
@@ -1640,19 +1694,37 @@ int sqlite3Fts5ExprClonePhrase(
    pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, 
        sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*));
  }
+  if( rc==SQLITE_OK ){
+    Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset;
+    if( pColsetOrig ){
+      int nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int);
+      Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte);
+      if( pColset ){ 
+        memcpy(pColset, pColsetOrig, nByte);
+      }
+      pNew->pRoot->pNear->pColset = pColset;
+    }
+  }

-  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
-    int tflags = 0;
-    Fts5ExprTerm *p;
-    for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
-      const char *zTerm = p->zTerm;
-      rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
-          0, 0);
-      tflags = FTS5_TOKEN_COLOCATED;
-    }
-    if( rc==SQLITE_OK ){
-      sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+  if( pOrig->nTerm ){
+    int i;                          /* Used to iterate through phrase terms */
+    for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
+      int tflags = 0;
+      Fts5ExprTerm *p;
+      for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
+        const char *zTerm = p->zTerm;
+        rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
+            0, 0);
+        tflags = FTS5_TOKEN_COLOCATED;
+      }
+      if( rc==SQLITE_OK ){
+        sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+      }
    }
+  }else{
+    /* This happens when parsing a token or quoted phrase that contains
+    ** no token characters at all. (e.g ... MATCH '""'). */
+    sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
  }

  if( rc==SQLITE_OK ){
@@ -1701,23 +1773,25 @@ void sqlite3Fts5ParseSetDistance(
  Fts5ExprNearset *pNear,
  Fts5Token *p
 ){
-  int nNear = 0;
-  int i;
-  if( p->n ){
-    for(i=0; i<p->n; i++){
-      char c = (char)p->p[i];
-      if( c<'0' || c>'9' ){
-        sqlite3Fts5ParseError(
-            pParse, "expected integer, got \"%.*s\"", p->n, p->p
-        );
-        return;
+  if( pNear ){
+    int nNear = 0;
+    int i;
+    if( p->n ){
+      for(i=0; i<p->n; i++){
+        char c = (char)p->p[i];
+        if( c<'0' || c>'9' ){
+          sqlite3Fts5ParseError(
+              pParse, "expected integer, got \"%.*s\"", p->n, p->p
+              );
+          return;
+        }
+        nNear = nNear * 10 + (p->p[i] - '0');
      }
-      nNear = nNear * 10 + (p->p[i] - '0');
+    }else{
+      nNear = FTS5_DEFAULT_NEARDIST;
    }
-  }else{
-    nNear = FTS5_DEFAULT_NEARDIST;
+    pNear->nNear = nNear;
  }
-  pNear->nNear = nNear;
 }

 /*
@@ -1765,6 +1839,34 @@ static Fts5Colset *fts5ParseColset(
  return pNew;
 }

+/*
+** Allocate and return an Fts5Colset object specifying the inverse of
+** the colset passed as the second argument. Free the colset passed
+** as the second argument before returning.
+*/
+Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){
+  Fts5Colset *pRet;
+  int nCol = pParse->pConfig->nCol;
+
+  pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, 
+      sizeof(Fts5Colset) + sizeof(int)*nCol
+  );
+  if( pRet ){
+    int i;
+    int iOld = 0;
+    for(i=0; i<nCol; i++){
+      if( iOld>=p->nCol || p->aiCol[iOld]!=i ){
+        pRet->aiCol[pRet->nCol++] = i;
+      }else{
+        iOld++;
+      }
+    }
+  }
+
+  sqlite3_free(p);
+  return pRet;
+}
+
 Fts5Colset *sqlite3Fts5ParseColset(
  Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  Fts5Colset *pColset,            /* Existing colset object */
@@ -1797,25 +1899,110 @@ Fts5Colset *sqlite3Fts5ParseColset(
  return pRet;
 }

+/*
+** If argument pOrig is NULL, or if (*pRc) is set to anything other than
+** SQLITE_OK when this function is called, NULL is returned. 
+**
+** Otherwise, a copy of (*pOrig) is made into memory obtained from
+** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
+** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
+*/
+static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
+  Fts5Colset *pRet;
+  if( pOrig ){
+    int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
+    pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
+    if( pRet ){ 
+      memcpy(pRet, pOrig, nByte);
+    }
+  }else{
+    pRet = 0;
+  }
+  return pRet;
+}
+
+/*
+** Remove from colset pColset any columns that are not also in colset pMerge.
+*/
+static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
+  int iIn = 0;          /* Next input in pColset */
+  int iMerge = 0;       /* Next input in pMerge */
+  int iOut = 0;         /* Next output slot in pColset */
+
+  while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
+    int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
+    if( iDiff==0 ){
+      pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
+      iMerge++;
+      iIn++;
+    }else if( iDiff>0 ){
+      iMerge++;
+    }else{
+      iIn++;
+    }
+  }
+  pColset->nCol = iOut;
+}
+
+/*
+** Recursively apply colset pColset to expression node pNode and all of
+** its decendents. If (*ppFree) is not NULL, it contains a spare copy
+** of pColset. This function may use the spare copy and set (*ppFree) to
+** zero, or it may create copies of pColset using fts5CloneColset().
+*/
+static void fts5ParseSetColset(
+  Fts5Parse *pParse, 
+  Fts5ExprNode *pNode, 
+  Fts5Colset *pColset,
+  Fts5Colset **ppFree
+){
+  if( pParse->rc==SQLITE_OK ){
+    assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING 
+         || pNode->eType==FTS5_AND  || pNode->eType==FTS5_OR
+         || pNode->eType==FTS5_NOT  || pNode->eType==FTS5_EOF
+    );
+    if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
+      Fts5ExprNearset *pNear = pNode->pNear;
+      if( pNear->pColset ){
+        fts5MergeColset(pNear->pColset, pColset);
+        if( pNear->pColset->nCol==0 ){
+          pNode->eType = FTS5_EOF;
+          pNode->xNext = 0;
+        }
+      }else if( *ppFree ){
+        pNear->pColset = pColset;
+        *ppFree = 0;
+      }else{
+        pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
+      }
+    }else{
+      int i;
+      assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
+      for(i=0; i<pNode->nChild; i++){
+        fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
+      }
+    }
+  }
+}
+
+/*
+** Apply colset pColset to expression node pExpr and all of its descendents.
+*/
 void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
-  Fts5ExprNearset *pNear, 
+  Fts5ExprNode *pExpr, 
  Fts5Colset *pColset 
 ){
+  Fts5Colset *pFree = pColset;
  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
    pParse->rc = SQLITE_ERROR;
    pParse->zErr = sqlite3_mprintf(
      "fts5: column queries are not supported (detail=none)"
    );
-    sqlite3_free(pColset);
-    return;
-  }
-
-  if( pNear ){
-    pNear->pColset = pColset;
  }else{
-    sqlite3_free(pColset);
+    fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
  }
+  sqlite3_free(pFree);
 }

 static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
@@ -1904,10 +2091,14 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
        int iPhrase;
        for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
          pNear->apPhrase[iPhrase]->pNode = pRet;
+          if( pNear->apPhrase[iPhrase]->nTerm==0 ){
+            pRet->xNext = 0;
+            pRet->eType = FTS5_EOF;
+          }
        }

        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL 
-         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm!=1)
+         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
        ){
          assert( pParse->rc==SQLITE_OK );
          pParse->rc = SQLITE_ERROR;
@@ -1936,6 +2127,70 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
  return pRet;
 }

+Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
+  Fts5Parse *pParse,              /* Parse context */
+  Fts5ExprNode *pLeft,            /* Left hand child expression */
+  Fts5ExprNode *pRight            /* Right hand child expression */
+){
+  Fts5ExprNode *pRet = 0;
+  Fts5ExprNode *pPrev;
+
+  if( pParse->rc ){
+    sqlite3Fts5ParseNodeFree(pLeft);
+    sqlite3Fts5ParseNodeFree(pRight);
+  }else{
+
+    assert( pLeft->eType==FTS5_STRING 
+        || pLeft->eType==FTS5_TERM
+        || pLeft->eType==FTS5_EOF
+        || pLeft->eType==FTS5_AND
+    );
+    assert( pRight->eType==FTS5_STRING 
+        || pRight->eType==FTS5_TERM 
+        || pRight->eType==FTS5_EOF 
+    );
+
+    if( pLeft->eType==FTS5_AND ){
+      pPrev = pLeft->apChild[pLeft->nChild-1];
+    }else{
+      pPrev = pLeft;
+    }
+    assert( pPrev->eType==FTS5_STRING 
+        || pPrev->eType==FTS5_TERM 
+        || pPrev->eType==FTS5_EOF 
+        );
+
+    if( pRight->eType==FTS5_EOF ){
+      assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
+      sqlite3Fts5ParseNodeFree(pRight);
+      pRet = pLeft;
+      pParse->nPhrase--;
+    }
+    else if( pPrev->eType==FTS5_EOF ){
+      Fts5ExprPhrase **ap;
+
+      if( pPrev==pLeft ){
+        pRet = pRight;
+      }else{
+        pLeft->apChild[pLeft->nChild-1] = pRight;
+        pRet = pLeft;
+      }
+
+      ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase];
+      assert( ap[0]==pPrev->pNear->apPhrase[0] );
+      memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase);
+      pParse->nPhrase--;
+
+      sqlite3Fts5ParseNodeFree(pPrev);
+    }
+    else{
+      pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0);
+    }
+  }
+
+  return pRet;
+}
+
 static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
  int nByte = 0;
  Fts5ExprTerm *p;
@@ -2070,6 +2325,9 @@ static char *fts5ExprPrintTcl(

 static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
  char *zRet = 0;
+  if( pExpr->eType==0 ){
+    return sqlite3_mprintf("\"\"");
+  }else
  if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){
    Fts5ExprNearset *pNear = pExpr->pNear;
    int i; 
@@ -2130,7 +2388,7 @@ static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
        zRet = 0;
      }else{
        int e = pExpr->apChild[i]->eType;
-        int b = (e!=FTS5_STRING && e!=FTS5_TERM);
+        int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF);
        zRet = fts5PrintfAppend(zRet, "%s%s%z%s", 
            (i==0 ? "" : zOp),
            (b?"(":""), z, (b?")":"")
@@ -2198,7 +2456,7 @@ static void fts5ExprFunction(

  rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  if( rc==SQLITE_OK ){
-    rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
+    rc = sqlite3Fts5ExprNew(pConfig, pConfig->nCol, zExpr, &pExpr, &zErr);
  }
  if( rc==SQLITE_OK ){
    char *zText;
@@ -2399,19 +2657,22 @@ static int fts5ExprPopulatePoslistsCb(
  int tflags,                /* Mask of FTS5_TOKEN_* flags */
  const char *pToken,        /* Pointer to buffer containing token */
  int nToken,                /* Size of token in bytes */
-  int iStart,                /* Byte offset of token within input text */
-  int iEnd                   /* Byte offset of end of token within input text */
+  int iUnused1,              /* Byte offset of token within input text */
+  int iUnused2               /* Byte offset of end of token within input text */
 ){
  Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
  Fts5Expr *pExpr = p->pExpr;
  int i;

+  UNUSED_PARAM2(iUnused1, iUnused2);
+
+  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
  for(i=0; i<pExpr->nPhrase; i++){
    Fts5ExprTerm *pTerm;
    if( p->aPopulator[i].bOk==0 ) continue;
    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
-      int nTerm = strlen(pTerm->zTerm);
+      int nTerm = (int)strlen(pTerm->zTerm);
      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
       && memcmp(pTerm->zTerm, pToken, nTerm)==0
      ){
@@ -2515,17 +2776,6 @@ void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
  fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
 }

-static void fts5ExprClearEof(Fts5ExprNode *pNode){
-  int i;
-  for(i=0; i<pNode->nChild; i++){
-    fts5ExprClearEof(pNode->apChild[i]);
-  }
-  pNode->bEof = 0;
-}
-void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
-  fts5ExprClearEof(pExpr->pRoot);
-}
-
 /*
 ** This function is only called for detail=columns tables. 
 */
@@ -2550,7 +2800,7 @@ int sqlite3Fts5ExprPhraseCollist(
    if( pTerm->pSynonym ){
      Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1];
      rc = fts5ExprSynonymList(
-          pTerm, 1, 0, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist
+          pTerm, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist
      );
    }else{
      *ppCollist = pPhrase->aTerm[0].pIter->pData;
@@ -36,9 +36,10 @@ struct Fts5Hash {

 /*
 ** Each entry in the hash table is represented by an object of the 
-** following type. Each object, its key (zKey[]) and its current data
-** are stored in a single memory allocation. The position list data 
-** immediately follows the key data in memory.
+** following type. Each object, its key (a nul-terminated string) and 
+** its current data are stored in a single memory allocation. The 
+** key immediately follows the object in memory. The position list
+** data immediately follows the key data in memory.
 **
 ** The data that follows the key is in a similar, but not identical format
 ** to the doclist data stored in the database. It is:
@@ -62,20 +63,20 @@ struct Fts5HashEntry {
  int nAlloc;                     /* Total size of allocation */
  int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
  int nData;                      /* Total bytes of data (incl. structure) */
+  int nKey;                       /* Length of key in bytes */
  u8 bDel;                        /* Set delete-flag @ iSzPoslist */
  u8 bContent;                    /* Set content-flag (detail=none mode) */
-
-  int iCol;                       /* Column of last value written */
+  i16 iCol;                       /* Column of last value written */
  int iPos;                       /* Position of last value written */
  i64 iRowid;                     /* Rowid of last value written */
-  char zKey[8];                   /* Nul-terminated entry key */
 };

 /*
-** Size of Fts5HashEntry without the zKey[] array.
+** Eqivalent to:
+**
+**   char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; }
 */
-#define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8)
-
+#define fts5EntryKey(p) ( ((char *)(&(p)[1])) )


 /*
@@ -170,10 +171,11 @@ static int fts5HashResize(Fts5Hash *pHash){

  for(i=0; i<pHash->nSlot; i++){
    while( apOld[i] ){
-      int iHash;
+      unsigned int iHash;
      Fts5HashEntry *p = apOld[i];
      apOld[i] = p->pHashNext;
-      iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey));
+      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
+                          (int)strlen(fts5EntryKey(p)));
      p->pHashNext = apNew[iHash];
      apNew[iHash] = p;
    }
@@ -244,9 +246,10 @@ int sqlite3Fts5HashWrite(
  /* Attempt to locate an existing hash entry */
  iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
-    if( p->zKey[0]==bByte 
-     && memcmp(&p->zKey[1], pToken, nToken)==0 
-     && p->zKey[nToken+1]==0 
+    char *zKey = fts5EntryKey(p);
+    if( zKey[0]==bByte 
+     && p->nKey==nToken
+     && memcmp(&zKey[1], pToken, nToken)==0 
    ){
      break;
    }
@@ -255,7 +258,8 @@ int sqlite3Fts5HashWrite(
  /* If an existing hash entry cannot be found, create a new one. */
  if( p==0 ){
    /* Figure out how much space to allocate */
-    int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
+    char *zKey;
+    int nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64;
    if( nByte<128 ) nByte = 128;

    /* Grow the Fts5Hash.aSlot[] array if necessary. */
@@ -268,13 +272,15 @@ int sqlite3Fts5HashWrite(
    /* Allocate new Fts5HashEntry and add it to the hash table. */
    p = (Fts5HashEntry*)sqlite3_malloc(nByte);
    if( !p ) return SQLITE_NOMEM;
-    memset(p, 0, FTS5_HASHENTRYSIZE);
+    memset(p, 0, sizeof(Fts5HashEntry));
    p->nAlloc = nByte;
-    p->zKey[0] = bByte;
-    memcpy(&p->zKey[1], pToken, nToken);
-    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
-    p->zKey[nToken+1] = '\0';
-    p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
+    zKey = fts5EntryKey(p);
+    zKey[0] = bByte;
+    memcpy(&zKey[1], pToken, nToken);
+    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
+    p->nKey = nToken;
+    zKey[nToken+1] = '\0';
+    p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
    p->pHashNext = pHash->aSlot[iHash];
    pHash->aSlot[iHash] = p;
    pHash->nEntry++;
@@ -344,11 +350,11 @@ int sqlite3Fts5HashWrite(
        if( pHash->eDetail==FTS5_DETAIL_FULL ){
          pPtr[p->nData++] = 0x01;
          p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
-          p->iCol = iCol;
+          p->iCol = (i16)iCol;
          p->iPos = 0;
        }else{
          bNew = 1;
-          p->iCol = iPos = iCol;
+          p->iCol = (i16)(iPos = iCol);
        }
      }

@@ -392,9 +398,11 @@ static Fts5HashEntry *fts5HashEntryMerge(
      p1 = 0;
    }else{
      int i = 0;
-      while( p1->zKey[i]==p2->zKey[i] ) i++;
+      char *zKey1 = fts5EntryKey(p1);
+      char *zKey2 = fts5EntryKey(p2);
+      while( zKey1[i]==zKey2[i] ) i++;

-      if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){
+      if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
        /* p2 is smaller */
        *ppOut = p2;
        ppOut = &p2->pScanNext;
@@ -437,7 +445,7 @@ static int fts5HashEntrySort(
  for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
    Fts5HashEntry *pIter;
    for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
-      if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){
+      if( pTerm==0 || 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm) ){
        Fts5HashEntry *pEntry = pIter;
        pEntry->pScanNext = 0;
        for(i=0; ap[i]; i++){
@@ -470,16 +478,18 @@ int sqlite3Fts5HashQuery(
  int *pnDoclist                  /* OUT: Size of doclist in bytes */
 ){
  unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
+  char *zKey = 0;
  Fts5HashEntry *p;

  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
-    if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break;
+    zKey = fts5EntryKey(p);
+    if( memcmp(zKey, pTerm, nTerm)==0 && zKey[nTerm]==0 ) break;
  }

  if( p ){
    fts5HashAddPoslistSize(pHash, p);
-    *ppDoclist = (const u8*)&p->zKey[nTerm+1];
-    *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
+    *ppDoclist = (const u8*)&zKey[nTerm+1];
+    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
  }else{
    *ppDoclist = 0;
    *pnDoclist = 0;
@@ -512,11 +522,12 @@ void sqlite3Fts5HashScanEntry(
 ){
  Fts5HashEntry *p;
  if( (p = pHash->pScan) ){
-    int nTerm = (int)strlen(p->zKey);
+    char *zKey = fts5EntryKey(p);
+    int nTerm = (int)strlen(zKey);
    fts5HashAddPoslistSize(pHash, p);
-    *pzTerm = p->zKey;
-    *ppDoclist = (const u8*)&p->zKey[nTerm+1];
-    *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
+    *pzTerm = zKey;
+    *ppDoclist = (const u8*)&zKey[nTerm+1];
+    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
  }else{
    *pzTerm = 0;
    *ppDoclist = 0;
@@ -506,6 +506,7 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
 static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  Fts5Table *pTab = (Fts5Table*)pVTab;
  Fts5Config *pConfig = pTab->pConfig;
+  const int nCol = pConfig->nCol;
  int idxFlags = 0;               /* Parameter passed through to xFilter() */
  int bHasMatch;
  int iNext;
@@ -531,24 +532,34 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){

  int aColMap[3];
  aColMap[0] = -1;
-  aColMap[1] = pConfig->nCol;
-  aColMap[2] = pConfig->nCol+1;
+  aColMap[1] = nCol;
+  aColMap[2] = nCol+1;

  /* Set idxFlags flags for all WHERE clause terms that will be used. */
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
-    int j;
-    for(j=0; j<ArraySize(aConstraint); j++){
-      struct Constraint *pC = &aConstraint[j];
-      if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){
-        if( p->usable ){
+    int iCol = p->iColumn;
+
+    if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol)
+     || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol==nCol)
+    ){
+      /* A MATCH operator or equivalent */
+      if( p->usable ){
+        idxFlags = (idxFlags & 0xFFFF) | FTS5_BI_MATCH | (iCol << 16);
+        aConstraint[0].iConsIndex = i;
+      }else{
+        /* As there exists an unusable MATCH constraint this is an 
+        ** unusable plan. Set a prohibitively high cost. */
+        pInfo->estimatedCost = 1e50;
+        return SQLITE_OK;
+      }
+    }else{
+      int j;
+      for(j=1; j<ArraySize(aConstraint); j++){
+        struct Constraint *pC = &aConstraint[j];
+        if( iCol==aColMap[pC->iCol] && p->op & pC->op && p->usable ){
          pC->iConsIndex = i;
          idxFlags |= pC->fts5op;
-        }else if( j==0 ){
-          /* As there exists an unusable MATCH constraint this is an 
-          ** unusable plan. Set a prohibitively high cost. */
-          pInfo->estimatedCost = 1e50;
-          return SQLITE_OK;
        }
      }
    }
@@ -597,27 +608,38 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  return SQLITE_OK;
 }

+static int fts5NewTransaction(Fts5Table *pTab){
+  Fts5Cursor *pCsr;
+  for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
+    if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
+  }
+  return sqlite3Fts5StorageReset(pTab->pStorage);
+}
+
 /*
 ** Implementation of xOpen method.
 */
 static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
  Fts5Table *pTab = (Fts5Table*)pVTab;
  Fts5Config *pConfig = pTab->pConfig;
-  Fts5Cursor *pCsr;               /* New cursor object */
+  Fts5Cursor *pCsr = 0;           /* New cursor object */
  int nByte;                      /* Bytes of space to allocate */
-  int rc = SQLITE_OK;             /* Return code */
+  int rc;                         /* Return code */

-  nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
-  pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
-  if( pCsr ){
-    Fts5Global *pGlobal = pTab->pGlobal;
-    memset(pCsr, 0, nByte);
-    pCsr->aColumnSize = (int*)&pCsr[1];
-    pCsr->pNext = pGlobal->pCsr;
-    pGlobal->pCsr = pCsr;
-    pCsr->iCsrId = ++pGlobal->iNextId;
-  }else{
-    rc = SQLITE_NOMEM;
+  rc = fts5NewTransaction(pTab);
+  if( rc==SQLITE_OK ){
+    nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
+    pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
+    if( pCsr ){
+      Fts5Global *pGlobal = pTab->pGlobal;
+      memset(pCsr, 0, nByte);
+      pCsr->aColumnSize = (int*)&pCsr[1];
+      pCsr->pNext = pGlobal->pCsr;
+      pGlobal->pCsr = pCsr;
+      pCsr->iCsrId = ++pGlobal->iNextId;
+    }else{
+      rc = SQLITE_NOMEM;
+    }
  }
  *ppCsr = (sqlite3_vtab_cursor*)pCsr;
  return rc;
@@ -861,7 +883,8 @@ static int fts5PrepareStatement(
  if( zSql==0 ){
    rc = SQLITE_NOMEM; 
  }else{
-    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
+    rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, 
+                            SQLITE_PREPARE_PERSISTENT, &pRet, 0);
    if( rc!=SQLITE_OK ){
      *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
    }
@@ -997,7 +1020,8 @@ static int fts5FindRankFunction(Fts5Cursor *pCsr){
    char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs);
    if( zSql ){
      sqlite3_stmt *pStmt = 0;
-      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0);
+      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
+                              SQLITE_PREPARE_PERSISTENT, &pStmt, 0);
      sqlite3_free(zSql);
      assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 );
      if( rc==SQLITE_OK ){
@@ -1096,7 +1120,7 @@ static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){
 static int fts5FilterMethod(
  sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  int idxNum,                     /* Strategy index */
-  const char *idxStr,             /* Unused */
+  const char *zUnused,            /* Unused */
  int nVal,                       /* Number of elements in apVal */
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
 ){
@@ -1112,8 +1136,12 @@ static int fts5FilterMethod(
  sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
+  int iCol;                       /* Column on LHS of MATCH operator */
  char **pzErrmsg = pConfig->pzErrmsg;

+  UNUSED_PARAM(zUnused);
+  UNUSED_PARAM(nVal);
+
  if( pCsr->ePlan ){
    fts5FreeCursorComponents(pCsr);
    memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr));
@@ -1139,6 +1167,8 @@ static int fts5FilterMethod(
  if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++];
  if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++];
  if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++];
+  iCol = (idxNum>>16);
+  assert( iCol>=0 && iCol<=pConfig->nCol );
  assert( iVal==nVal );
  bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0);
  pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0);
@@ -1172,7 +1202,6 @@ static int fts5FilterMethod(
    pCsr->ePlan = FTS5_PLAN_SOURCE;
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bDesc);
-    sqlite3Fts5ExprClearEof(pCsr->pExpr);
  }else if( pMatch ){
    const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
    if( zExpr==0 ) zExpr = "";
@@ -1186,7 +1215,7 @@ static int fts5FilterMethod(
        rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
      }else{
        char **pzErr = &pTab->base.zErrMsg;
-        rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr);
+        rc = sqlite3Fts5ExprNew(pConfig, iCol, zExpr, &pCsr->pExpr, pzErr);
        if( rc==SQLITE_OK ){
          if( bOrderByRank ){
            pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
@@ -1398,8 +1427,7 @@ static int fts5SpecialInsert(

 static int fts5SpecialDelete(
  Fts5Table *pTab, 
-  sqlite3_value **apVal, 
-  sqlite3_int64 *piRowid
+  sqlite3_value **apVal
 ){
  int rc = SQLITE_OK;
  int eType1 = sqlite3_value_type(apVal[1]);
@@ -1475,7 +1503,7 @@ static int fts5UpdateMethod(
    if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
      && 0==sqlite3_stricmp("delete", z) 
    ){
-      rc = fts5SpecialDelete(pTab, apVal, pRowid);
+      rc = fts5SpecialDelete(pTab, apVal);
    }else{
      rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
    }
@@ -1509,13 +1537,13 @@ static int fts5UpdateMethod(
      rc = SQLITE_ERROR;
    }

-    /* Case 1: DELETE */
+    /* DELETE */
    else if( nArg==1 ){
      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
    }

-    /* Case 2: INSERT */
+    /* INSERT */
    else if( eType0!=SQLITE_INTEGER ){     
      /* If this is a REPLACE, first remove the current entry (if any) */
      if( eConflict==SQLITE_REPLACE 
@@ -1527,7 +1555,7 @@ static int fts5UpdateMethod(
      fts5StorageInsert(&rc, pTab, apVal, pRowid);
    }

-    /* Case 2: UPDATE */
+    /* UPDATE */
    else{
      i64 iOld = sqlite3_value_int64(apVal[0]);  /* Old rowid */
      i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
@@ -1567,7 +1595,7 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
  fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
  pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;
  fts5TripCursors(pTab);
-  rc = sqlite3Fts5StorageSync(pTab->pStorage, 1);
+  rc = sqlite3Fts5StorageSync(pTab->pStorage);
  pTab->pConfig->pzErrmsg = 0;
  return rc;
 }
@@ -1577,6 +1605,7 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
 */
 static int fts5BeginMethod(sqlite3_vtab *pVtab){
  fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0);
+  fts5NewTransaction((Fts5Table*)pVtab);
  return SQLITE_OK;
 }

@@ -1586,6 +1615,7 @@ static int fts5BeginMethod(sqlite3_vtab *pVtab){
 ** by fts5SyncMethod().
 */
 static int fts5CommitMethod(sqlite3_vtab *pVtab){
+  UNUSED_PARAM(pVtab);  /* Call below is a no-op for NDEBUG builds */
  fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_COMMIT, 0);
  return SQLITE_OK;
 }
@@ -1839,12 +1869,14 @@ static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
 static int fts5ColumnSizeCb(
  void *pContext,                 /* Pointer to int */
  int tflags,
-  const char *pToken,             /* Buffer containing token */
-  int nToken,                     /* Size of token in bytes */
-  int iStart,                     /* Start offset of token */
-  int iEnd                        /* End offset of token */
+  const char *pUnused,            /* Buffer containing token */
+  int nUnused,                    /* Size of token in bytes */
+  int iUnused1,                   /* Start offset of token */
+  int iUnused2                    /* End offset of token */
 ){
  int *pCnt = (int*)pContext;
+  UNUSED_PARAM2(pUnused, nUnused);
+  UNUSED_PARAM2(iUnused1, iUnused2);
  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
    (*pCnt)++;
  }
@@ -1960,10 +1992,11 @@ static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){
 }

 static void fts5ApiPhraseNext(
-  Fts5Context *pCtx, 
+  Fts5Context *pUnused, 
  Fts5PhraseIter *pIter, 
  int *piCol, int *piOff
 ){
+  UNUSED_PARAM(pUnused);
  if( pIter->a>=pIter->b ){
    *piCol = -1;
    *piOff = -1;
@@ -2115,12 +2148,11 @@ static int fts5ApiQueryPhrase(

  rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew);
  if( rc==SQLITE_OK ){
-    Fts5Config *pConf = pTab->pConfig;
    pNew->ePlan = FTS5_PLAN_MATCH;
    pNew->iFirstRowid = SMALLEST_INT64;
    pNew->iLastRowid = LARGEST_INT64;
    pNew->base.pVtab = (sqlite3_vtab*)pTab;
-    rc = sqlite3Fts5ExprClonePhrase(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr);
+    rc = sqlite3Fts5ExprClonePhrase(pCsr->pExpr, iPhrase, &pNew->pExpr);
  }

  if( rc==SQLITE_OK ){
@@ -2333,7 +2365,7 @@ static int fts5ColumnMethod(
 */
 static int fts5FindFunctionMethod(
  sqlite3_vtab *pVtab,            /* Virtual table handle */
-  int nArg,                       /* Number of SQL function arguments */
+  int nUnused,                    /* Number of SQL function arguments */
  const char *zName,              /* Name of SQL function */
  void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
  void **ppArg                    /* OUT: User data for *pxFunc */
@@ -2341,6 +2373,7 @@ static int fts5FindFunctionMethod(
  Fts5Table *pTab = (Fts5Table*)pVtab;
  Fts5Auxiliary *pAux;

+  UNUSED_PARAM(nUnused);
  pAux = fts5FindAuxiliary(pTab, zName);
  if( pAux ){
    *pxFunc = fts5ApiCallback;
@@ -2370,9 +2403,10 @@ static int fts5RenameMethod(
 */
 static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5Table *pTab = (Fts5Table*)pVtab;
+  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
  fts5TripCursors(pTab);
-  return sqlite3Fts5StorageSync(pTab->pStorage, 0);
+  return sqlite3Fts5StorageSync(pTab->pStorage);
 }

 /*
@@ -2382,9 +2416,10 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
 */
 static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5Table *pTab = (Fts5Table*)pVtab;
+  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
  fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
  fts5TripCursors(pTab);
-  return sqlite3Fts5StorageSync(pTab->pStorage, 0);
+  return sqlite3Fts5StorageSync(pTab->pStorage);
 }

 /*
@@ -2394,6 +2429,7 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
 */
 static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5Table *pTab = (Fts5Table*)pVtab;
+  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
  fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
  fts5TripCursors(pTab);
  return sqlite3Fts5StorageRollback(pTab->pStorage);
@@ -2573,10 +2609,11 @@ static void fts5ModuleDestroy(void *pCtx){
 static void fts5Fts5Func(
  sqlite3_context *pCtx,          /* Function call context */
  int nArg,                       /* Number of args */
-  sqlite3_value **apVal           /* Function arguments */
+  sqlite3_value **apUnused        /* Function arguments */
 ){
  Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
  char buf[8];
+  UNUSED_PARAM2(nArg, apUnused);
  assert( nArg==0 );
  assert( sizeof(buf)>=sizeof(pGlobal) );
  memcpy(buf, (void*)&pGlobal, sizeof(pGlobal));
@@ -2589,9 +2626,10 @@ static void fts5Fts5Func(
 static void fts5SourceIdFunc(
  sqlite3_context *pCtx,          /* Function call context */
  int nArg,                       /* Number of args */
-  sqlite3_value **apVal           /* Function arguments */
+  sqlite3_value **apUnused        /* Function arguments */
 ){
  assert( nArg==0 );
+  UNUSED_PARAM2(nArg, apUnused);
  sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT);
 }

@@ -2653,6 +2691,17 @@ static int fts5Init(sqlite3 *db){
      );
    }
  }
+
+  /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file
+  ** fts5_test_mi.c is compiled and linked into the executable. And call
+  ** its entry point to enable the matchinfo() demo.  */
+#ifdef SQLITE_FTS5_ENABLE_TEST_MI
+  if( rc==SQLITE_OK ){
+    extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*);
+    rc = sqlite3Fts5TestRegisterMatchinfo(db);
+  }
+#endif
+
  return rc;
 }

@@ -136,7 +136,8 @@ static int fts5StorageGetStmt(
    if( zSql==0 ){
      rc = SQLITE_NOMEM;
    }else{
-      rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
+      rc = sqlite3_prepare_v3(pC->db, zSql, -1,
+                              SQLITE_PREPARE_PERSISTENT, &p->aStmt[eStmt], 0);
      sqlite3_free(zSql);
      if( rc!=SQLITE_OK && pzErrMsg ){
        *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
@@ -145,6 +146,7 @@ static int fts5StorageGetStmt(
  }

  *ppStmt = p->aStmt[eStmt];
+  sqlite3_reset(*ppStmt);
  return rc;
 }

@@ -217,7 +219,7 @@ static void fts5StorageRenameOne(

 int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
  Fts5Config *pConfig = pStorage->pConfig;
-  int rc = sqlite3Fts5StorageSync(pStorage, 1);
+  int rc = sqlite3Fts5StorageSync(pStorage);

  fts5StorageRenameOne(pConfig, &rc, "data", zName);
  fts5StorageRenameOne(pConfig, &rc, "idx", zName);
@@ -246,7 +248,11 @@ int sqlite3Fts5CreateTable(
  char *zErr = 0;

  rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s",
-      pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":""
+      pConfig->zDb, pConfig->zName, zPost, zDefn, 
+#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID
+      bWithout?" WITHOUT ROWID":
+#endif
+      ""
  );
  if( zErr ){
    *pzErr = sqlite3_mprintf(
@@ -362,11 +368,13 @@ static int fts5StorageInsertCallback(
  int tflags,
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
-  int iStart,                     /* Start offset of token */
-  int iEnd                        /* End offset of token */
+  int iUnused1,                   /* Start offset of token */
+  int iUnused2                    /* End offset of token */
 ){
  Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
  Fts5Index *pIdx = pCtx->pStorage->pIndex;
+  UNUSED_PARAM2(iUnused1, iUnused2);
+  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
  if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
    pCtx->szCol++;
  }
@@ -538,11 +546,6 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
    }
  }

-  /* Write the averages record */
-  if( rc==SQLITE_OK ){
-    rc = fts5StorageSaveTotals(p);
-  }
-
  return rc;
 }

@@ -638,6 +641,10 @@ int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
  return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
 }

+int sqlite3Fts5StorageReset(Fts5Storage *p){
+  return sqlite3Fts5IndexReset(p->pIndex);
+}
+
 /*
 ** Allocate a new rowid. This is used for "external content" tables when
 ** a NULL value is inserted into the rowid column. The new rowid is allocated
@@ -742,11 +749,6 @@ int sqlite3Fts5StorageIndexInsert(
  }
  sqlite3_free(buf.p);

-  /* Write the averages record */
-  if( rc==SQLITE_OK ){
-    rc = fts5StorageSaveTotals(p);
-  }
-
  return rc;
 }

@@ -797,8 +799,8 @@ static int fts5StorageIntegrityCallback(
  int tflags,
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
-  int iStart,                     /* Start offset of token */
-  int iEnd                        /* End offset of token */
+  int iUnused1,                   /* Start offset of token */
+  int iUnused2                    /* End offset of token */
 ){
  Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
  Fts5Termset *pTermset = pCtx->pTermset;
@@ -808,6 +810,9 @@ static int fts5StorageIntegrityCallback(
  int iPos;
  int iCol;

+  UNUSED_PARAM2(iUnused1, iUnused2);
+  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
+
  if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
    pCtx->szCol++;
  }
@@ -1077,13 +1082,18 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
 /*
 ** Flush any data currently held in-memory to disk.
 */
-int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){
-  if( bCommit && p->bTotalsValid ){
-    int rc = fts5StorageSaveTotals(p);
+int sqlite3Fts5StorageSync(Fts5Storage *p){
+  int rc = SQLITE_OK;
+  i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
+  if( p->bTotalsValid ){
+    rc = fts5StorageSaveTotals(p);
    p->bTotalsValid = 0;
-    if( rc!=SQLITE_OK ) return rc;
  }
-  return sqlite3Fts5IndexSync(p->pIndex, bCommit);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexSync(p->pIndex);
+  }
+  sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid);
+  return rc;
 }

 int sqlite3Fts5StorageRollback(Fts5Storage *p){
@@ -1118,5 +1128,3 @@ int sqlite3Fts5StorageConfigValue(
  }
  return rc;
 }
-
-
@@ -14,7 +14,14 @@


 #ifdef SQLITE_TEST
-#include <tcl.h>
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif

 #ifdef SQLITE_ENABLE_FTS5

@@ -78,7 +85,7 @@ static int f5tResultToErrorCode(const char *zRes){
  return SQLITE_ERROR;
 }

-static int f5tDbAndApi(
+static int SQLITE_TCLAPI f5tDbAndApi(
  Tcl_Interp *interp, 
  Tcl_Obj *pObj, 
  sqlite3 **ppDb, 
@@ -164,7 +171,7 @@ static int xTokenizeCb(
  return rc;
 }

-static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);
+static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);

 static int xQueryPhraseCb(
  const Fts5ExtensionApi *pApi, 
@@ -209,7 +216,7 @@ static void xSetAuxdataDestructor(void *p){
 **
 ** Description...
 */
-static int xF5tApi(
+static int SQLITE_TCLAPI xF5tApi(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -602,7 +609,7 @@ static void xF5tDestroy(void *pCtx){
 **
 ** Description...
 */
-static int f5tCreateFunction(
+static int SQLITE_TCLAPI f5tCreateFunction(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -672,7 +679,7 @@ static int xTokenizeCb2(
 **
 ** Description...
 */
-static int f5tTokenize(
+static int SQLITE_TCLAPI f5tTokenize(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -878,7 +885,7 @@ static int f5tTokenizerTokenize(
 /*
 ** sqlite3_fts5_token ?-colocated? TEXT START END
 */
-static int f5tTokenizerReturn(
+static int SQLITE_TCLAPI f5tTokenizerReturn(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -949,7 +956,7 @@ static void f5tDelTokenizer(void *pCtx){
 ** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each
 ** token within the tokenized text.
 */
-static int f5tCreateTokenizer(
+static int SQLITE_TCLAPI f5tCreateTokenizer(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
@@ -992,7 +999,7 @@ static int f5tCreateTokenizer(
  return TCL_OK;
 }

-static void xF5tFree(ClientData clientData){
+static void SQLITE_TCLAPI xF5tFree(ClientData clientData){
  ckfree(clientData);
 }

@@ -1001,7 +1008,7 @@ static void xF5tFree(ClientData clientData){
 **
 ** Set or clear the global "may-be-corrupt" flag. Return the old value.
 */
-static int f5tMayBeCorrupt(
+static int SQLITE_TCLAPI f5tMayBeCorrupt(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -1033,7 +1040,7 @@ static unsigned int f5t_fts5HashKey(int nSlot, const char *p, int n){
  return (h % nSlot);
 }

-static int f5tTokenHash(
+static int SQLITE_TCLAPI f5tTokenHash(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -1058,7 +1065,7 @@ static int f5tTokenHash(
  return TCL_OK;
 }

-static int f5tRegisterMatchinfo(
+static int SQLITE_TCLAPI f5tRegisterMatchinfo(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -1083,7 +1090,7 @@ static int f5tRegisterMatchinfo(
  return TCL_OK;
 }

-static int f5tRegisterTok(
+static int SQLITE_TCLAPI f5tRegisterTok(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
@@ -41,16 +41,17 @@
 */


-#ifdef SQLITE_TEST
 #ifdef SQLITE_ENABLE_FTS5

 #include "fts5.h"
-#include <tcl.h>
 #include <assert.h>
 #include <string.h>

 typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx;
+
+#ifndef SQLITE_AMALGAMATION
 typedef unsigned int u32;
+#endif

 struct Fts5MatchinfoCtx {
  int nCol;                       /* Number of cols in FTS5 table */
@@ -67,18 +68,22 @@ struct Fts5MatchinfoCtx {
 ** If an error occurs, return NULL and leave an error in the database 
 ** handle (accessible using sqlite3_errcode()/errmsg()).
 */
-static fts5_api *fts5_api_from_db(sqlite3 *db){
-  fts5_api *pRet = 0;
+static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
  sqlite3_stmt *pStmt = 0;
+  int rc;

-  if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0)
-   && SQLITE_ROW==sqlite3_step(pStmt) 
-   && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0)
-  ){
-    memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet));
+  *ppApi = 0;
+  rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0);
+  if( rc==SQLITE_OK ){
+    if( SQLITE_ROW==sqlite3_step(pStmt) 
+        && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0)
+      ){
+      memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*));
+    }
+    rc = sqlite3_finalize(pStmt);
  }
-  sqlite3_finalize(pStmt);
-  return pRet;
+
+  return rc;
 }


@@ -244,11 +249,7 @@ static int fts5MatchinfoLocalCb(
            iOff>=0; 
            pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
        ){
-          if( f=='b' ){
-            aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
-          }else{
-            aOut[nMul * (iCol + iPhrase * p->nCol)]++;
-          }
+          aOut[nMul * (iCol + iPhrase * p->nCol)]++;
        }
      }

@@ -402,14 +403,15 @@ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){
  /* Extract the FTS5 API pointer from the database handle. The 
  ** fts5_api_from_db() function above is copied verbatim from the 
  ** FTS5 documentation. Refer there for details. */
-  pApi = fts5_api_from_db(db);
+  rc = fts5_api_from_db(db, &pApi);
+  if( rc!=SQLITE_OK ) return rc;

  /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
  ** with this database handle, or an error (OOM perhaps?) has occurred.
  **
  ** Also check that the fts5_api object is version 2 or newer.  
  */ 
-  if( pApi==0 || pApi->iVersion<1 ){
+  if( pApi==0 || pApi->iVersion<2 ){
    return SQLITE_ERROR;
  }

@@ -420,5 +422,4 @@ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){
 }

 #endif /* SQLITE_ENABLE_FTS5 */
-#endif /* SQLITE_TEST */

@@ -40,7 +40,7 @@
 */
 #if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)

-#include <fts5.h>
+#include "fts5.h"
 #include <string.h>
 #include <assert.h>

@@ -182,7 +182,7 @@ static int fts5tokConnectMethod(
  Fts5tokTable *pTab = 0;
  int rc;
  char **azDequote = 0;
-  int nDequote;
+  int nDequote = 0;

  rc = sqlite3_declare_vtab(db, 
       "CREATE TABLE x(input HIDDEN, token, start, end, position)"
@@ -62,12 +62,13 @@ static void fts5AsciiDelete(Fts5Tokenizer *p){
 ** Create an "ascii" tokenizer.
 */
 static int fts5AsciiCreate(
-  void *pCtx, 
+  void *pUnused, 
  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
 ){
  int rc = SQLITE_OK;
  AsciiTokenizer *p = 0;
+  UNUSED_PARAM(pUnused);
  if( nArg%2 ){
    rc = SQLITE_ERROR;
  }else{
@@ -116,7 +117,7 @@ static void asciiFold(char *aOut, const char *aIn, int nByte){
 static int fts5AsciiTokenize(
  Fts5Tokenizer *pTokenizer,
  void *pCtx,
-  int flags,
+  int iUnused,
  const char *pText, int nText,
  int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
 ){
@@ -130,6 +131,8 @@ static int fts5AsciiTokenize(
  char *pFold = aFold;
  unsigned char *a = p->aTokenChar;

+  UNUSED_PARAM(iUnused);
+
  while( is<nText && rc==SQLITE_OK ){
    int nByte;

@@ -323,13 +326,15 @@ static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
 ** Create a "unicode61" tokenizer.
 */
 static int fts5UnicodeCreate(
-  void *pCtx, 
+  void *pUnused, 
  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
 ){
  int rc = SQLITE_OK;             /* Return code */
  Unicode61Tokenizer *p = 0;      /* New tokenizer object */ 

+  UNUSED_PARAM(pUnused);
+
  if( nArg%2 ){
    rc = SQLITE_ERROR;
  }else{
@@ -386,7 +391,7 @@ static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
 static int fts5UnicodeTokenize(
  Fts5Tokenizer *pTokenizer,
  void *pCtx,
-  int flags,
+  int iUnused,
  const char *pText, int nText,
  int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
 ){
@@ -402,6 +407,8 @@ static int fts5UnicodeTokenize(
  int nFold = p->nFold;
  const char *pEnd = &aFold[nFold-6];

+  UNUSED_PARAM(iUnused);
+
  /* Each iteration of this loop gobbles up a contiguous run of separators,
  ** then the next token.  */
  while( rc==SQLITE_OK ){
@@ -125,9 +125,9 @@ int sqlite3Fts5UnicodeIsalnum(int c){
    0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
  };

-  if( c<128 ){
+  if( (unsigned int)c<128 ){
    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
-  }else if( c<(1<<22) ){
+  }else if( (unsigned int)c<(1<<22) ){
    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
    int iRes = 0;
    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
@@ -237,7 +237,7 @@ static int fts5VocabCreateMethod(
 ** Implementation of the xBestIndex method.
 */
 static int fts5VocabBestIndexMethod(
-  sqlite3_vtab *pVTab, 
+  sqlite3_vtab *pUnused,
  sqlite3_index_info *pInfo
 ){
  int i;
@@ -247,6 +247,8 @@ static int fts5VocabBestIndexMethod(
  int idxNum = 0;
  int nArg = 0;

+  UNUSED_PARAM(pUnused);
+
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
    if( p->usable==0 ) continue;
@@ -277,8 +279,19 @@ static int fts5VocabBestIndexMethod(
    }
  }

-  pInfo->idxNum = idxNum;
+  /* This virtual table always delivers results in ascending order of
+  ** the "term" column (column 0). So if the user has requested this
+  ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the
+  ** sqlite3_index_info.orderByConsumed flag to tell the core the results
+  ** are already in sorted order.  */
+  if( pInfo->nOrderBy==1 
+   && pInfo->aOrderBy[0].iColumn==0 
+   && pInfo->aOrderBy[0].desc==0
+  ){
+    pInfo->orderByConsumed = 1;
+  }

+  pInfo->idxNum = idxNum;
  return SQLITE_OK;
 }

@@ -488,8 +501,8 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
 static int fts5VocabFilterMethod(
  sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  int idxNum,                     /* Strategy index */
-  const char *idxStr,             /* Unused */
-  int nVal,                       /* Number of elements in apVal */
+  const char *zUnused,            /* Unused */
+  int nUnused,                    /* Number of elements in apVal */
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
 ){
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
@@ -504,6 +517,8 @@ static int fts5VocabFilterMethod(
  sqlite3_value *pGe = 0;
  sqlite3_value *pLe = 0;

+  UNUSED_PARAM2(zUnused, nUnused);
+
  fts5VocabResetCursor(pCsr);
  if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
  if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
@@ -28,12 +28,13 @@
 // This code runs whenever there is a syntax error
 //
 %syntax_error {
+  UNUSED_PARAM(yymajor); /* Silence a compiler warning */
  sqlite3Fts5ParseError(
    pParse, "fts5: syntax error near \"%.*s\"",TOKEN.n,TOKEN.p
  );
 }
 %stack_overflow {
-  assert( 0 );
+  sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow");
 }

 // The name of the generated procedure that implements the parser
@@ -88,6 +89,29 @@ input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); }
 %destructor expr     { sqlite3Fts5ParseNodeFree($$); }
 %destructor exprlist { sqlite3Fts5ParseNodeFree($$); }

+%type colset {Fts5Colset*}
+%destructor colset { sqlite3_free($$); }
+%type colsetlist {Fts5Colset*}
+%destructor colsetlist { sqlite3_free($$); }
+
+colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
+    A = sqlite3Fts5ParseColsetInvert(pParse, X);
+}
+colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
+colset(A) ::= STRING(X). {
+  A = sqlite3Fts5ParseColset(pParse, 0, &X);
+}
+colset(A) ::= MINUS STRING(X). {
+  A = sqlite3Fts5ParseColset(pParse, 0, &X);
+  A = sqlite3Fts5ParseColsetInvert(pParse, A);
+}
+
+colsetlist(A) ::= colsetlist(Y) STRING(X). { 
+  A = sqlite3Fts5ParseColset(pParse, Y, &X); }
+colsetlist(A) ::= STRING(X). { 
+  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
+}
+
 expr(A) ::= expr(X) AND expr(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
 }
@@ -98,36 +122,24 @@ expr(A) ::= expr(X) NOT expr(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
 }

+expr(A) ::= colset(X) COLON LP expr(Y) RP. {
+  sqlite3Fts5ParseSetColset(pParse, Y, X);
+  A = Y;
+}
 expr(A) ::= LP expr(X) RP. {A = X;}
 expr(A) ::= exprlist(X).   {A = X;}

 exprlist(A) ::= cnearset(X). {A = X;}
 exprlist(A) ::= exprlist(X) cnearset(Y). {
-  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
+  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
 }

 cnearset(A) ::= nearset(X). { 
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
 }
 cnearset(A) ::= colset(X) COLON nearset(Y). { 
-  sqlite3Fts5ParseSetColset(pParse, Y, X);
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
-}
-
-%type colset {Fts5Colset*}
-%destructor colset { sqlite3_free($$); }
-%type colsetlist {Fts5Colset*}
-%destructor colsetlist { sqlite3_free($$); }
-
-colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
-colset(A) ::= STRING(X). {
-  A = sqlite3Fts5ParseColset(pParse, 0, &X);
-}
-
-colsetlist(A) ::= colsetlist(Y) STRING(X). { 
-  A = sqlite3Fts5ParseColset(pParse, Y, &X); }
-colsetlist(A) ::= STRING(X). { 
-  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
+  sqlite3Fts5ParseSetColset(pParse, A, X);
 }


@@ -16,8 +16,13 @@ if {![info exists testdir]} {
 source $testdir/tester.tcl

 ifcapable !fts5 {
-  finish_test
+  proc return_if_no_fts5 {} {
+    finish_test
+    return -code return
+  }
  return
+} else {
+  proc return_if_no_fts5 {} {}
 }

 catch { 
@@ -25,12 +30,6 @@ catch {
  reset_db
 }

-# If SQLITE_ENABLE_FTS5 is not defined, skip this test.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
 proc fts5_test_poslist {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
@@ -160,6 +159,12 @@ proc fts5_aux_test_functions {db} {
  }
 }

+proc fts5_segcount {tbl} {
+  set N 0
+  foreach n [fts5_level_segs $tbl] { incr N $n }
+  set N
+}
+
 proc fts5_level_segs {tbl} {
  set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
  set ret [list]
@@ -432,9 +432,16 @@ proc funk {} {
 }
 db func funk funk

+# This test case corrupts the structure record within the first invocation
+# of function funk(). Which used to cause the bm25() function to throw an
+# exception. But since bm25() can now used the cached structure record,
+# it never sees the corruption introduced by funk() and so the following 
+# statement no longer fails.
+#
 do_catchsql_test 16.2 {
  SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
-} {1 {SQL logic error or missing database}}
+} {0 {{} -1e-06 {}}}
+# {1 {SQL logic error}}

 #-------------------------------------------------------------------------
 #
@@ -554,9 +561,37 @@ do_test 20.1 {
  execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
 } $::ids

+#--------------------------------------------------------------------
+# Test that a DROP TABLE may be executed within a transaction that
+# writes to an FTS5 table.
+#
+do_execsql_test 21.0 {
+  CREATE TEMP TABLE t8(a, b);
+  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
+}
+
+do_execsql_test 21.1 {
+  BEGIN;
+    INSERT INTO ft VALUES('a b c');
+    DROP TABLE t8;
+  COMMIT;
+}
+
+do_execsql_test 22.0 {
+  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
+  INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
+  BEGIN;
+    INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
+    DELETE FROM t9 WHERE rowid = 2;
+    INSERT INTO t9(rowid, x) VALUES(3, 'bbb');
+  COMMIT;
+}
+
+do_execsql_test 22.1 {
+  SELECT rowid FROM t9('a*')
+} {1}
+
 }


 finish_test
-
-
@@ -294,4 +294,3 @@ do_execsql_test 7.0 {


 finish_test
-
@@ -276,4 +276,3 @@ foreach {tn expr tclexpr} {
 }

 finish_test
-
@@ -242,4 +242,3 @@ foreach {T create} {
 }

 finish_test
-
@@ -309,4 +309,3 @@ foreach {tn q cnt} {
 }

 finish_test
-
@@ -72,42 +72,56 @@ foreach {tn doc res} {
  2.2 {o X o o o o o o} {o [X] o o o o o...}
  2.3 {o o X o o o o o} {o o [X] o o o o...}
  2.4 {o o o X o o o o} {o o o [X] o o o...}
-  2.5 {o o o o X o o o} {...o o o [X] o o o}
-  2.6 {o o o o o X o o} {...o o o o [X] o o}
-  2.7 {o o o o o o X o} {...o o o o o [X] o}
+  2.5 {o o o o X o o o} {o o o o [X] o o...}
+  2.6 {o o o o o X o o} {o o o o o [X] o...}
+  2.7 {o o o o o o X o} {o o o o o o [X]...}
  2.8 {o o o o o o o X} {...o o o o o o [X]}

+  2.9  {o o o o o o o X o}       {...o o o o o [X] o}
+  2.10 {o o o o o o o X o o}     {...o o o o [X] o o}
+  2.11 {o o o o o o o X o o o}   {...o o o [X] o o o}
+  2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...}
+
+
  3.1 {X o o o o o o o o} {[X] o o o o o o...}
  3.2 {o X o o o o o o o} {o [X] o o o o o...}
  3.3 {o o X o o o o o o} {o o [X] o o o o...}
  3.4 {o o o X o o o o o} {o o o [X] o o o...}
-  3.5 {o o o o X o o o o} {...o o o [X] o o o...}
-  3.6 {o o o o o X o o o} {...o o o [X] o o o}
-  3.7 {o o o o o o X o o} {...o o o o [X] o o}
-  3.8 {o o o o o o o X o} {...o o o o o [X] o}
-  3.9 {o o o o o o o o X} {...o o o o o o [X]}
+
+  3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...}
+  3.6 {o o o o o o o o X o o o} {...o o o [X] o o o}
+  3.7 {o o o o o o o o o X o o} {...o o o o [X] o o}
+  3.8 {o o o o o o o o o o X o} {...o o o o o [X] o}
+  3.9 {o o o o o o o o o o o X} {...o o o o o o [X]}

  4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
-  4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
-  4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
+  4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...}
+  4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]}

  5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
-  5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
-  5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
-  5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
+  5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...}
+  5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o}
+  5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]}

  6.1 {X o o o X o o o} {[X] o o o [X] o o...}
  6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
-  6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
-  6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
-  6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
+  6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...}
+  6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o}
+  6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]}

  7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
  7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
-  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
-  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
-  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
-  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
+  7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...}
+  7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o}
+  7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o}
+  7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]}
+
+  8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o}
+      {...o o [X] [X] [X] o o...}
+  8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} 
+      {...o o [X] o o o o...}
+  8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
+      {o o o o [X] o o...}
 } {
  do_snippet_test 1.$tn $doc X $res
 }
@@ -124,25 +138,43 @@ if {[detail_is_full]} {
    2.1 {X Y o o o o o o} {[X Y] o o o o o...}
    2.2 {o X Y o o o o o} {o [X Y] o o o o...}
    2.3 {o o X Y o o o o} {o o [X Y] o o o...}
-    2.4 {o o o X Y o o o} {...o o [X Y] o o o}
-    2.5 {o o o o X Y o o} {...o o o [X Y] o o}
-    2.6 {o o o o o X Y o} {...o o o o [X Y] o}
-    2.7 {o o o o o o X Y} {...o o o o o [X Y]}
+    2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o}
+    2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o}
+    2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o}
+    2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]}

    3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
    3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
    3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
-    3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
-    3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
-    3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
-    3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
-    3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
+    3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...}
+    3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o}
+    3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o}
+    3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o}
+    3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]}
  } {
    do_snippet_test 2.$tn $doc "X + Y" $res
  }
 }

+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+  INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13');
+  SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz');
+} {
+  {1 2 3 4 5...}
+}
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE p1 USING fts5(a, b);
+  INSERT INTO p1 VALUES(
+    'x a a a a a a a a a a',
+    'a a a a a a a a a a a a a a a a a a a x'
+  );
+}
+do_execsql_test 5.1 {
+  SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
+} {{[x] a a a a a...}}
+
 } ;# foreach_detail_mode 

 finish_test
-
@@ -142,4 +142,3 @@ if {[detail_is_full]} {


 finish_test
-
@@ -167,4 +167,3 @@ do_execsql_test 1.8.2 {
 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}

 finish_test
-
@@ -55,4 +55,3 @@ do_execsql_test 1.2 {


 finish_test
-
@@ -66,4 +66,3 @@ do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }


 finish_test
-
@@ -147,4 +147,3 @@ do_execsql_test 3.1 {
 }

 finish_test
-
@@ -77,7 +77,7 @@ foreach {tn defn} {
 } {
  do_test 2.2.$tn {
    catchsql { INSERT INTO ft1(ft1, rank) VALUES('rank', $defn) }
-  } {1 {SQL logic error or missing database}}
+  } {1 {SQL logic error}}
 }

 #-------------------------------------------------------------------------
@@ -297,4 +297,3 @@ do_catchsql_test 4.4.4 {


 finish_test
-
@@ -100,4 +100,3 @@ do_execsql_test 3.3 {
 } {1 2}

 finish_test
-
@@ -342,4 +342,3 @@ foreach {tn expr} {


 finish_test
-
@@ -246,5 +246,36 @@ foreach {tn lRow res} {
  } $res
 }

-finish_test
+#-------------------------------------------------------------------------
+# Test the built-in bm25() demo.
+#
+reset_db
+do_execsql_test 9.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+  INSERT INTO t1 VALUES('a',   NULL);           -- 1
+  INSERT INTO t1 VALUES('a',   NULL);           -- 2
+  INSERT INTO t1 VALUES('a',   NULL);           -- 3
+  INSERT INTO t1 VALUES('a',   NULL);           -- 4
+  INSERT INTO t1 VALUES('a',   NULL);           -- 5
+  INSERT INTO t1 VALUES('a',   NULL);           -- 6
+  INSERT INTO t1 VALUES('a',   NULL);           -- 7
+  INSERT INTO t1 VALUES('a',   NULL);           -- 8
+  INSERT INTO t1 VALUES(NULL,  'a a b');        -- 9
+  INSERT INTO t1 VALUES(NULL,  'b b a');        -- 10
+}

+do_execsql_test 9.2 {
+  SELECT rowid FROM t1('a AND b') ORDER BY rank;
+} {
+  10 9
+}
+
+do_execsql_test 9.3 {
+  SELECT rowid FROM t1('b:a AND b:b') ORDER BY rank;
+} {
+  9 10
+}
+
+
+
+finish_test
@@ -112,4 +112,3 @@ db eval {
 }

 finish_test
-
@@ -61,4 +61,3 @@ do_test 2.1...slow {
 } {}

 finish_test
-
@@ -14,6 +14,7 @@

 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5bigtok
+return_if_no_fts5

 proc rndterm {} {
  set L [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
@@ -63,5 +64,3 @@ foreach_detail_mode $::testprefix {
 }

 finish_test
-
-
@@ -0,0 +1,86 @@
+# 2016 August 10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5colset
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $::testprefix {
+  if {[detail_is_none]} continue
+
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d, detail=%DETAIL%);
+    INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
+    INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
+    INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
+    INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
+  }
+
+  foreach {tn q res} {
+    1 "a"          {1 2 3 4}
+    2 "{a}   : a"  {1}
+    3 "-{a}   : a" {2 3 4}
+    4 "- {a c} : a" {2 4}
+    5 " - {d d c} : a" {1 2}
+    6 "- {d c b a} : a" {}
+    7 "-{\"a\"} : b" {1 2 3}
+    8 "- c : a" {1 2 4}
+    9 "-c : a"  {1 2 4}
+    10 "-\"c\" : a"  {1 2 4}
+  } {
+    do_execsql_test 1.$tn {
+      SELECT rowid FROM t1($q)
+    } $res
+  }
+
+  foreach {tn q res} {
+    0 {{a} : (a AND ":")}     {}
+    1 "{a b c} : (a AND d)"   {2 3}
+    2 "{a b c} : (a AND b:d)" {3}
+    3 "{a b c} : (a AND d:d)" {}
+    4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
+    5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
+    6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
+    7 "{a b c} : (b:a AND c:b)" {2}
+  } {
+    do_execsql_test 2.$tn {
+      SELECT rowid FROM t1($q)
+    } $res
+  }
+
+  foreach {tn w res} {
+    0 "a MATCH 'a'" {1}
+    1 "b MATCH 'a'" {2}
+    2 "b MATCH '{a b c} : a'" {2}
+    3 "b MATCH 'a OR b'"      {1 2}
+    4 "b MATCH 'a OR a:b'"    {2}
+    5 "b MATCH 'a OR b:b'"    {1 2}
+  } {
+    do_execsql_test 3.$tn "
+      SELECT rowid FROM t1 WHERE $w
+    " $res
+  }
+
+  do_catchsql_test 4.1 {
+    SELECT * FROM t1 WHERE rowid MATCH 'a'
+  } {1 {unable to use function MATCH in the requested context}}
+}
+
+
+finish_test
@@ -66,7 +66,7 @@ foreach {tn val} {
 } {
  do_catchsql_test 3.$tn {
    INSERT INTO t1(t1, rank) VALUES('rank', $val);
-  } {1 {SQL logic error or missing database}}
+  } {1 {SQL logic error}}
 }

 #-------------------------------------------------------------------------
@@ -169,33 +169,33 @@ do_execsql_test 9.0 {
 } {}
 do_catchsql_test 9.1.1 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', -5);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.1.2 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.1.3 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}

 do_catchsql_test 9.2.1 {
  INSERT INTO abc(abc, rank) VALUES('automerge', -5);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.2.2 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 50000000);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.2.3 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 66.67);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_execsql_test 9.2.4 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 1);
 } {}

 do_catchsql_test 9.3.1 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', -5);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.3.2 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_execsql_test 9.3.3 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 1);
 } {}
@@ -205,14 +205,14 @@ do_execsql_test 9.3.4 {

 do_catchsql_test 9.4.1 {
  INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}

 do_catchsql_test 9.5.1 {
  INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.5.2 {
  INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.5.3 {
  INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
 } {0 {}}
@@ -245,7 +245,22 @@ foreach {tn opt} {

 do_catchsql_test 12.1 {
  INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
+
+#-------------------------------------------------------------------------
+# errors in the 'usermerge' option
+#
+do_execsql_test 13.0 {
+  CREATE VIRTUAL TABLE tt USING fts5(ttt);
+}
+foreach {tn val} {
+  1     -1
+  2     4.2
+  3     17
+  4     1
+} {
+  set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)"
+  do_catchsql_test 13.$tn $sql {1 {SQL logic error}}
+}

 finish_test
-
@@ -66,5 +66,3 @@ do_execsql_test 2.1 {
 }

 finish_test
-
-
@@ -255,4 +255,3 @@ do_execsql_test 6.2 {


 finish_test
-
@@ -96,4 +96,3 @@ do_catchsql_test 3.1 {
 } {1 {database disk image is malformed}}

 finish_test
-
@@ -37,7 +37,7 @@ do_execsql_test 1.0 {
 }
 set mask [expr 31 << 31]

-if 1 {
+if 0 {

 # Test 1:
 #
@@ -84,6 +84,8 @@ foreach {tno stmt} {
  }
 }

+}
+
 # Using the same database as the 1.* tests.
 #
 # Run N-1 tests, where N is the number of bytes in the rightmost leaf page
@@ -212,8 +214,6 @@ foreach {tn nCut} {
  # do_test 4.$tn.x { expr $nCorrupt>0 } 1
 }

-}
-
 set doc [string repeat "A B C " 1000]
 do_execsql_test 5.0 {
  CREATE VIRTUAL TABLE x5 USING fts5(tt);
@@ -269,4 +269,3 @@ do_catchsql_test 6.2 {

 sqlite3_fts5_may_be_corrupt 0
 finish_test
-
@@ -179,6 +179,10 @@ for {set i 1} {1} {incr i} {
  if {$end<=$i} break
  lset var end [expr $end - $i]
  set struct [binary format c* $var]
+
+  db close
+  sqlite3 db test.db
+
  db eval {
    BEGIN;
    UPDATE t1_data SET block = $struct WHERE id=10;
@@ -334,12 +338,9 @@ do_catchsql_test 6.3.5 {
 } {1 {database disk image is malformed}}


-}
-
 #------------------------------------------------------------------------
 #
 reset_db
-reset_db
 proc rnddoc {n} {
  set map [list a b c d]
  set doc [list]
@@ -371,6 +372,40 @@ do_test 7.1 {
  }
 } {}

+}
+
+#------------------------------------------------------------------------
+# Corruption within the structure record.
+#
+reset_db
+do_execsql_test 8.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+  INSERT INTO t1 VALUES('one', 'two');
+}
+
+do_test 9.1.1 {
+  set    blob "12345678"    ;# cookie
+  append blob "0105"        ;# 1 level, total of 5 segments
+  append blob "06"          ;# write counter
+  append blob "0002"        ;# first level has 0 segments merging, 2 other.
+  append blob "450108"      ;# first segment
+  execsql "REPLACE INTO t1_data VALUES(10, X'$blob')"
+} {}
+do_catchsql_test 9.1.2 {
+  SELECT * FROM t1('one AND two');
+} {1 {database disk image is malformed}}
+
+do_test 9.2.1 {
+  set    blob "12345678"    ;# cookie
+  append blob "0205"        ;# 2 levels, total of 5 segments
+  append blob "06"          ;# write counter
+  append blob "0001"        ;# first level has 0 segments merging, 1 other.
+  append blob "450108"      ;# first segment
+  execsql "REPLACE INTO t1_data VALUES(10, X'$blob')"
+} {}
+do_catchsql_test 9.2.2 {
+  SELECT * FROM t1('one AND two');
+} {1 {database disk image is malformed}}
+
 sqlite3_fts5_may_be_corrupt 0
 finish_test
-
@@ -0,0 +1,53 @@
+# 2017 May 12
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5delete
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+fts5_aux_test_functions db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<5000
+  )
+  INSERT INTO t1(rowid, x) SELECT i, (i/2)*2 FROM s;
+}
+
+do_test 1.1 {
+  execsql BEGIN
+  for {set i 1} {$i<=5000} {incr i} {
+    if {$i % 2} {
+      execsql { INSERT INTO t1 VALUES($i) }
+    } else {
+      execsql { DELETE FROM t1 WHERE rowid = $i }
+    }
+  }
+  execsql COMMIT
+} {}
+
+do_test 1.2 {
+  execsql { INSERT INTO t1(t1, rank) VALUES('usermerge', 2); }
+  for {set i 0} {$i < 5} {incr i} {
+    execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+  }
+} {}
+
+finish_test
@@ -241,4 +241,3 @@ do_execsql_test 5.3 {


 finish_test
-
@@ -0,0 +1,65 @@
+# 2016 March 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+# Specifically, that the fts5 module is deterministic. At one point, when
+# segment ids were allocated using sqlite3_randomness(), this was not the
+# case.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5aa
+return_if_no_fts5 
+
+proc do_determin_test {tn} {
+  uplevel [list
+    do_execsql_test $tn {
+      SELECT (SELECT md5sum(id, block) FROM t1_data)==
+             (SELECT md5sum(id, block) FROM t2_data),
+             (SELECT md5sum(id, block) FROM t1_data)==
+             (SELECT md5sum(id, block) FROM t3_data)
+    } {1 1}
+  ]
+}
+
+foreach_detail_mode $::testprefix {
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
+    CREATE VIRTUAL TABLE t2 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
+    CREATE VIRTUAL TABLE t3 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
+  }
+
+  do_test 1.1 {
+    foreach t {t1 t2 t3} {
+      execsql [string map [list TBL $t] {
+        INSERT INTO TBL VALUES('a b c', 'd e f');
+        INSERT INTO TBL VALUES('c1 c2 c3', 'c1 c2 c3');
+        INSERT INTO TBL VALUES('xyzxyzxyz', 'xyzxyzxyz');
+      }]
+    }
+  } {}
+
+  do_determin_test 1.2
+
+  do_test 1.3 {
+    foreach t {t1 t2 t3} {
+      execsql [string map [list TBL $t] {
+        INSERT INTO TBL(TBL) VALUES('optimize');
+      }]
+    }
+  } {}
+
+  do_determin_test 1.4
+}
+
+
+finish_test
@@ -178,7 +178,7 @@ do_execsql_test 3.2 {
  ORDER BY rowid DESC;
 } {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1}

-do_execsql_test 3.2 {
+do_execsql_test 3.3 {
  INSERT INTO abc(abc) VALUES('integrity-check');
  INSERT INTO abc(abc) VALUES('optimize');
  INSERT INTO abc(abc) VALUES('integrity-check');
@@ -187,7 +187,7 @@ do_execsql_test 3.2 {
 set v [lindex $vocab 0]
 set i 0
 foreach v $vocab {
-  do_execsql_test 3.3.[incr i] {
+  do_execsql_test 3.4.[incr i] {
    SELECT rowid FROM abc WHERE abc MATCH $v
  } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
 }
@@ -197,4 +197,3 @@ foreach v $vocab {


 finish_test
-
@@ -44,4 +44,3 @@ do_execsql_test 1.2 {


 finish_test
-
@@ -33,12 +33,12 @@ foreach {tn expr res} {
  1  {abc}                            {"abc"}
  2  {abc ""}                         {"abc"}
  3  {""}                             {}
-  4  {abc OR ""}                      {"abc"}
-  5  {abc NOT ""}                     {"abc"}
-  6  {abc AND ""}                     {"abc"}
-  7  {"" OR abc}                      {"abc"}
-  8  {"" NOT abc}                     {"abc"}
-  9  {"" AND abc}                     {"abc"}
+  4  {abc OR ""}                      {"abc" OR ""}
+  5  {abc NOT ""}                     {"abc" NOT ""}
+  6  {abc AND ""}                     {"abc" AND ""}
+  7  {"" OR abc}                      {"" OR "abc"}
+  8  {"" NOT abc}                     {"" NOT "abc"}
+  9  {"" AND abc}                     {"" AND "abc"}
  10 {abc + "" + def}                 {"abc" + "def"}
  11 {abc "" def}                     {"abc" AND "def"}
  12 {r+e OR w}                       {"r" + "e" OR "w"}
@@ -63,7 +63,21 @@ do_catchsql_test 2.1 {
  SELECT fts5_expr_tcl()
 } {1 {wrong number of arguments to function fts5_expr_tcl}}

+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE e1 USING fts5(text, tokenize = 'porter unicode61');
+  INSERT INTO e1 VALUES ("just a few words with a / inside");
+}
+do_execsql_test 3.1 {
+  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
+} {1 -1e-06}
+do_execsql_test 3.2 {
+  SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"'
+} 1
+do_execsql_test 3.3 {
+  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
+} {1 -1e-06}
+
+
+
 finish_test
-
-
-
@@ -351,4 +351,3 @@ do_faultsim_test 9.1 -faults oom-* -prep {


 finish_test
-
@@ -137,4 +137,3 @@ do_faultsim_test 5.0 -faults oom-* -prep {
 }

 finish_test
-
@@ -110,4 +110,3 @@ do_faultsim_test 3.2 -faults oom-* -prep {


 finish_test
-
@@ -86,7 +86,7 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]
 do_faultsim_test 4 -faults oom-* -body {
  db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
 } -test {
-  faultsim_test_result {0 {0 {} 4}}
+  faultsim_test_result {0 {0 {} 3}}
 }

 #-------------------------------------------------------------------------
@@ -395,4 +395,3 @@ do_faultsim_test 14.1 -faults oom-t* -prep {
 }

 finish_test
-
@@ -130,4 +130,3 @@ do_faultsim_test 3.3 -faults oom-t* -body {


 finish_test
-
@@ -292,4 +292,3 @@ do_faultsim_test 6 -faults oom* -prep {
  db close
 }
 finish_test
-
@@ -116,4 +116,3 @@ do_faultsim_test 2.2 -faults oom-* -body {
 }

 finish_test
-
@@ -54,7 +54,31 @@ foreach_detail_mode $testprefix {
      faultsim_test_result {0 {1 3}} {1 SQLITE_NOMEM}
    }
  }
+
 } ;# foreach_detail_mode...

-finish_test

+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x2 USING fts5(a);
+  INSERT INTO x2(x2, rank) VALUES('crisismerge', 2);
+  INSERT INTO x2(x2, rank) VALUES('pgsz', 32);
+  INSERT INTO x2 VALUES('a b c d');
+  INSERT INTO x2 VALUES('e f g h');
+  INSERT INTO x2 VALUES('i j k l');
+  INSERT INTO x2 VALUES('m n o p');
+  INSERT INTO x2 VALUES('q r s t');
+  INSERT INTO x2 VALUES('u v w x');
+  INSERT INTO x2 VALUES('y z a b');
+}
+faultsim_save_and_close
+
+do_faultsim_test 4 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { INSERT INTO x2(x2) VALUES('optimize') }
+} -test {
+  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
+}
+
+
+finish_test
@@ -153,4 +153,3 @@ do_faultsim_test 6 -faults oom-* -body {
 } ;# foreach_detail_mode...

 finish_test
-
@@ -61,4 +61,3 @@ do_faultsim_test 2 -faults oom* -prep {
  faultsim_test_result {0 {1 2}}
 }
 finish_test
-
@@ -0,0 +1,134 @@
+# 2016 February 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultB
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc mit {blob} {
+  set scan(littleEndian) i*
+  set scan(bigEndian) I*
+  binary scan $blob $scan($::tcl_platform(byteOrder)) r
+  return $r
+}
+db func mit mit
+
+
+#-------------------------------------------------------------------------
+# Errors while registering the matchinfo() demo function.
+#
+do_faultsim_test 1 -faults oom* -prep {
+  sqlite3 db test.db
+} -body {
+  sqlite3_fts5_register_matchinfo db
+} -test {
+  faultsim_test_result {0 {}} {1 SQLITE_ERROR} {1 SQLITE_NOMEM}
+}
+
+
+#-------------------------------------------------------------------------
+# Errors while executing the matchinfo() demo function.
+#
+reset_db
+sqlite3_fts5_register_matchinfo db
+db func mit mit
+do_execsql_test 2 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+  INSERT INTO t1 VALUES('x y z', '1 2 3');
+  INSERT INTO t1 VALUES('x', '1 2 3 4 5 6 7');
+}
+
+do_faultsim_test 2.1 -faults oom* -body {
+  execsql { SELECT mit(matchinfo(t1, 'a')) FROM t1('x') }
+} -test {
+  faultsim_test_result {0 {{2 5} {2 5}}} 
+}
+
+do_faultsim_test 2.2 -faults oom* -body {
+  execsql { SELECT mit(matchinfo(t1, 'l')) FROM t1('x') }
+} -test {
+  faultsim_test_result {0 {{3 3} {1 7}}} 
+}
+
+do_execsql_test 2.3 {
+  INSERT INTO t1 VALUES('a b c d e f', 'a b d e f c');
+  INSERT INTO t1 VALUES('l m b c a', 'n o a b c z');
+}
+
+do_faultsim_test 2.4 -faults oom* -body {
+  execsql { SELECT mit(matchinfo(t1, 's')) FROM t1('a b c') }
+} -test {
+  faultsim_test_result {0 {{3 2} {2 3}}} 
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db 
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(z);
+}
+
+do_faultsim_test 3.1 -faults oom* -body {
+  execsql {
+    SELECT rowid FROM x1('c') WHERE rowid>1;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_execsql_test 3.2 {
+  INSERT INTO x1 VALUES('a b c');
+  INSERT INTO x1 VALUES('b c d');
+  INSERT INTO x1 VALUES('c d e');
+  INSERT INTO x1 VALUES('d e f');
+}
+do_faultsim_test 3.3 -faults oom* -body {
+  execsql {
+    SELECT rowid FROM x1('c') WHERE rowid>1;
+  }
+} -test {
+  faultsim_test_result {0 {2 3}}
+}
+
+#-------------------------------------------------------------------------
+# Test OOM injection with nested colsets.
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d);
+  INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
+  INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
+  INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
+  INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
+}
+do_faultsim_test 4.1 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); }
+} -test {
+  faultsim_test_result {0 2}
+}
+
+do_faultsim_test 4.2 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
+} -test {
+  faultsim_test_result {0 {2 3}}
+}
+
+
+finish_test
@@ -0,0 +1,87 @@
+# 2016 February 2
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultA
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+  if {"%DETAIL%"=="none"} continue
+
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE o1 USING fts5(a, b, c, detail=%DETAIL%);
+    INSERT INTO o1(o1, rank) VALUES('pgsz', 32);
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
+    INSERT INTO o1 SELECT 'A', 'B', 'C' FROM s;
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
+    INSERT INTO o1 SELECT 'C', 'A', 'B' FROM s;
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
+    INSERT INTO o1 SELECT 'B', 'C', 'A' FROM s;
+  }
+  
+  do_faultsim_test 1 -faults int* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT count(*) FROM o1('a') }
+  } -test {
+    faultsim_test_result {0 180} {1 {vtable constructor failed: o1}}
+  }
+
+  do_faultsim_test 2 -faults int* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT * FROM o1('a:a AND {b c}:b') ORDER BY rank }
+    expr 1
+  } -test {
+    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
+  }
+
+  do_faultsim_test 3 -faults int* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT * FROM o1('{b c}:b NOT a:a') ORDER BY rank }
+    expr 1
+  } -test {
+    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
+  }
+
+  do_faultsim_test 4 -faults int* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT * FROM o1('b:b OR a:a') }
+    expr 1
+  } -test {
+    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
+  }
+
+  do_faultsim_test 5 -faults int* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT count(*) FROM o1('c:b') }
+    expr 1
+  } -test {
+    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
+  }
+}
+
+finish_test
@@ -40,4 +40,3 @@ do_test 1.1 {


 finish_test
-
@@ -0,0 +1,92 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+return_if_no_fts5
+set testprefix fts5fuzz1
+
+
+#-------------------------------------------------------------------------
+reset_db
+do_catchsql_test 1.1 {
+  CREATE VIRTUAL TABLE f1 USING fts5(a b);
+} {/1 {parse error in.*}/}
+
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE f1 USING fts5(a, b);
+  INSERT INTO f1 VALUES('a b', 'c d');
+  INSERT INTO f1 VALUES('e f', 'a b');
+}
+
+do_execsql_test 2.2.1 {
+  SELECT rowid FROM f1('""');
+} {}
+
+do_execsql_test 2.2.2 {
+  SELECT rowid FROM f1('"" AND a');
+} {}
+
+
+do_execsql_test 2.2.3 {
+  SELECT rowid FROM f1('"" a');
+} {1 2}
+
+do_execsql_test 2.2.4 {
+  SELECT rowid FROM f1('"" OR a');
+} {1 2}
+
+do_execsql_test 2.3 {
+  SELECT a, b FROM f1('NEAR("")');
+} {}
+
+do_execsql_test 2.4 {
+  SELECT a, b FROM f1('NEAR("", 5)');
+} {}
+
+do_execsql_test 2.5 {
+  SELECT a, b FROM f1('NEAR("" c, 5)');
+} {{a b} {c d}}
+
+do_execsql_test 2.6 {
+  SELECT a, b FROM f1('NEAR("" c d, 5)');
+} {{a b} {c d}}
+
+do_execsql_test 2.7 {
+  SELECT a, b FROM f1('NEAR(c d, 5)');
+} {{a b} {c d}}
+
+do_execsql_test 2.8 {
+  SELECT rowid FROM f1('NEAR("a" "b", 5)');
+} {1 2}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.2 {
+  CREATE VIRTUAL TABLE f2 USING fts5(o, t, tokenize="ascii separators abc");
+  SELECT * FROM f2('a+4');
+} {}
+
+
+
+#-------------------------------------------------------------------------
+reset_db
+do_catchsql_test 4.1 {
+  CREATE VIRTUAL TABLE f2 USING fts5(o, t);
+  SELECT * FROM f2('(8 AND 9)`AND 10');
+} {1 {fts5: syntax error near "`"}}
+
+finish_test
@@ -66,49 +66,67 @@ proc random_doc {vocab nWord} {

 foreach_detail_mode $testprefix {

-set vocab [build_vocab1]
-db func r random_doc 
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
-  BEGIN;
-    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
-    INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
-    INSERT INTO eee(eee) VALUES('integrity-check');
-  COMMIT;
-  INSERT INTO eee(eee) VALUES('integrity-check');
-}
-
-set hash [sqlite3_fts5_token_hash 1024 xyz]
-set vocab [build_vocab1 -prefix xyz -hash $hash]
-lappend vocab xyz
-
-do_execsql_test 1.1 {
-  CREATE VIRTUAL TABLE vocab USING fts5vocab(eee, 'row'); 
-  BEGIN;
-}
-do_test 1.2 {
-  for {set i 1} {$i <= 100} {incr i} {
-    execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) }
-  }
-} {}
+  set vocab [build_vocab1]
+  db func r random_doc 
  
-do_test 1.3 {
-  db eval { SELECT term, doc FROM vocab } {
-    set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
-    if {$nRow != $doc} {
-      error "term=$term fts5vocab=$doc cnt=$nRow"
-    }
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
+    BEGIN;
+      WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
+      INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
+      INSERT INTO eee(eee) VALUES('integrity-check');
+    COMMIT;
+    INSERT INTO eee(eee) VALUES('integrity-check');
+  }
+  
+  set hash [sqlite3_fts5_token_hash 1024 xyz]
+  set vocab [build_vocab1 -prefix xyz -hash $hash]
+  lappend vocab xyz
+  
+  do_execsql_test 1.1 {
+    CREATE VIRTUAL TABLE vocab USING fts5vocab(eee, 'row'); 
+    BEGIN;
+  }
+  do_test 1.2 {
+    for {set i 1} {$i <= 100} {incr i} {
+      execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) }
+    }
+  } {}
+    
+  do_test 1.3 {
+    db eval { SELECT term, doc FROM vocab } {
+      set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
+      if {$nRow != $doc} {
+        error "term=$term fts5vocab=$doc cnt=$nRow"
+      }
+    }
+    set {} {}
+  } {}
+  
+  do_execsql_test 1.4 {
+    COMMIT;
+    INSERT INTO eee(eee) VALUES('integrity-check');
  }
-  set {} {}
-} {}

-do_execsql_test 1.4 {
-  COMMIT;
-  INSERT INTO eee(eee) VALUES('integrity-check');
-}
+  #-----------------------------------------------------------------------
+  # Add a small and very large token with the same hash value to an
+  # empty table. At one point this would provoke an asan error.
+  #
+  do_test 2.0 {
+    set big [string repeat 12345 40]
+    set hash [sqlite3_fts5_token_hash 1024 $big]
+    while {1} {
+      set small [random_token]
+      if {[sqlite3_fts5_token_hash 1024 $small]==$hash} break
+    }
+
+    execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) }
+breakpoint
+    execsql {
+      INSERT INTO t2 VALUES($small || ' ' || $big);
+    }
+  } {}

 } ;# foreach_detail_mode

 finish_test
-
@@ -210,4 +210,3 @@ foreach {tn pgsz} {
 }

 finish_test
-
@@ -0,0 +1,72 @@
+# 2017 Feb 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests of the last_insert_rowid functionality with fts5.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5lastrowid
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(str);
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO t1 VALUES('one string');
+  INSERT INTO t1 VALUES('two string');
+  INSERT INTO t1 VALUES('three string');
+  SELECT last_insert_rowid();
+} {3}
+
+do_execsql_test 1.2 {
+  BEGIN;
+    INSERT INTO t1 VALUES('one string');
+    INSERT INTO t1 VALUES('two string');
+    INSERT INTO t1 VALUES('three string');
+  COMMIT;
+  SELECT last_insert_rowid();
+} {6}
+
+do_execsql_test 1.3 {
+  INSERT INTO t1(rowid, str) VALUES(-22, 'some more text');
+  SELECT last_insert_rowid();
+} {-22}
+
+do_execsql_test 1.4 {
+  BEGIN;
+    INSERT INTO t1(rowid, str) VALUES(45, 'some more text');
+    INSERT INTO t1(rowid, str) VALUES(46, 'some more text');
+    INSERT INTO t1(rowid, str) VALUES(222, 'some more text');
+    SELECT last_insert_rowid();
+  COMMIT;
+  SELECT last_insert_rowid();
+} {222 222}
+
+do_execsql_test 1.5 {
+  CREATE TABLE x1(x);
+  INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo');
+  INSERT INTO t1 SELECT x FROM x1;
+  SELECT last_insert_rowid();
+} {226}
+
+do_execsql_test 1.6 {
+  INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1;
+  SELECT last_insert_rowid();
+} {14}
+
+
+finish_test
@@ -0,0 +1,43 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5leftjoin
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE vt USING fts5(x);
+  INSERT INTO vt VALUES('abc');
+  INSERT INTO vt VALUES('xyz');
+
+  CREATE TABLE t1(a INTEGER PRIMARY KEY);
+  INSERT INTO t1 VALUES(1), (2);
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM t1 LEFT JOIN (
+    SELECT rowid AS rrr, * FROM vt WHERE vt MATCH 'abc'
+  ) ON t1.a = rrr
+} {1 1 abc 2 {} {}}
+
+do_execsql_test 1.2 {
+  SELECT * FROM t1 LEFT JOIN vt ON (vt MATCH 'abc')
+} {1 abc 2 abc}
+
+finish_test
@@ -467,5 +467,28 @@ do_execsql_test 12.1 {

 } ;# foreach_detail_mode

-finish_test
+#-------------------------------------------------------------------------
+# Test that a bad fts5() return is detected
+#
+reset_db
+proc xyz {} {}
+db func fts5 -argcount 0 xyz
+do_test 13.1 {
+  list [catch { sqlite3_fts5_register_matchinfo db } msg] $msg
+} {1 SQLITE_ERROR}

+#-------------------------------------------------------------------------
+# Test that an invalid matchinfo() flag is detected
+#
+reset_db
+sqlite3_fts5_register_matchinfo db
+do_execsql_test 14.1 {
+  CREATE VIRTUAL TABLE x1 USING fts5(z);
+  INSERT INTO x1 VALUES('a b c a b c a b c');
+} {}
+
+do_catchsql_test 14.2 {
+  SELECT matchinfo(x1, 'd') FROM x1('a b c');
+} {1 {unrecognized matchinfo flag: d}}
+
+finish_test
@@ -45,7 +45,7 @@ proc do_merge1_test {testname nRowPerSeg} {
    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<$::nRowPerSeg)
      INSERT INTO x8 SELECT repeat('x y ', i % 16) FROM ii;

-    INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+    INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
  }

  for {set tn 1} {[lindex [fts5_level_segs x8] 0]>0} {incr tn} {
@@ -84,9 +84,9 @@ proc do_merge2_test {testname nRow} {
      execsql { INSERT INTO x8 VALUES( rnddoc(($i%16) + 5) ) }
      while {[not_merged x8]} {
        execsql {
-          INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+          INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
          INSERT INTO x8(x8, rank) VALUES('merge', 1);
-          INSERT INTO x8(x8, rank) VALUES('automerge', 16);
+          INSERT INTO x8(x8, rank) VALUES('usermerge', 16);
          INSERT INTO x8(x8) VALUES('integrity-check');
        }
      }
@@ -104,9 +104,9 @@ do_merge2_test 2.2   10
 do_merge2_test 2.3   20

 #-------------------------------------------------------------------------
-# Test that an auto-merge will complete any merge that has already been
+# Test that a merge will complete any merge that has already been
 # started, even if the number of input segments is less than the current
-# value of the 'automerge' configuration parameter.
+# value of the 'usermerge' configuration parameter.
 #
 db func rnddoc fts5_rnddoc

@@ -119,7 +119,7 @@ do_execsql_test 3.1 {
 }
 do_test 3.2 {
  execsql {
-    INSERT INTO x8(x8, rank) VALUES('automerge', 4);
+    INSERT INTO x8(x8, rank) VALUES('usermerge', 4);
    INSERT INTO x8(x8, rank) VALUES('merge', 1);
  }
  fts5_level_segs x8
@@ -127,14 +127,14 @@ do_test 3.2 {

 do_test 3.3 {
  execsql {
-    INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+    INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
    INSERT INTO x8(x8, rank) VALUES('merge', 1);
  }
  fts5_level_segs x8
 } {2 1}

 do_test 3.4 {
-  execsql { INSERT INTO x8(x8, rank) VALUES('automerge', 4) }
+  execsql { INSERT INTO x8(x8, rank) VALUES('usermerge', 4) }
  while {[not_merged x8]} {
    execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1) }
  }
@@ -176,7 +176,7 @@ foreach {tn pgsz} {
      INSERT INTO x8 SELECT mydoc() FROM ii;
    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
      INSERT INTO x8 SELECT mydoc() FROM ii;
-    INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+    INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
  }

  set expect [mycount]
@@ -190,5 +190,54 @@ foreach {tn pgsz} {
 #  db eval {SELECT fts5_decode(rowid, block) AS r FROM x8_data} { puts $r }
 }

-finish_test
+#-------------------------------------------------------------------------
+# Test that the 'merge' command does not modify the database if there is
+# no work to do. 

+do_execsql_test 5.1 {
+  CREATE VIRTUAL TABLE x9 USING fts5(one, two);
+  INSERT INTO x9(x9, rank) VALUES('pgsz', 32);
+  INSERT INTO x9(x9, rank) VALUES('automerge', 2);
+  INSERT INTO x9(x9, rank) VALUES('usermerge', 2);
+  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+}
+
+do_test 5.2 {
+  while 1 {
+    set nChange [db total_changes]
+    execsql { INSERT INTO x9(x9, rank) VALUES('merge', 1); }
+    set nChange [expr [db total_changes] - $nChange]
+    #puts $nChange
+    if {$nChange<2} break
+  }
+} {}
+
+
+#--------------------------------------------------------------------------
+# Test that running 'merge' on an empty database does not cause a 
+# problem.
+#
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE g1 USING fts5(a, b);
+}
+do_execsql_test 6.1 {
+  INSERT INTO g1(g1, rank) VALUES('merge', 10);
+}
+do_execsql_test 6.2 {
+  INSERT INTO g1(g1, rank) VALUES('merge', -10);
+}
+do_execsql_test 6.3 {
+  INSERT INTO g1(g1) VALUES('integrity-check');
+}
+
+
+
+finish_test
@@ -13,7 +13,8 @@
 #

 source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5merge
+set testprefix fts5merge2
+return_if_no_fts5

 proc dump_structure {} {
  db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
@@ -26,8 +27,6 @@ proc dump_structure {} {

 foreach_detail_mode $testprefix {

-if {[detail_is_none]==0} continue
-
 do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@@ -56,4 +55,3 @@ do_execsql_test 1.2 {
 }

 finish_test
-
@@ -0,0 +1,47 @@
+# 2016 March 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/lock_common.tcl
+
+set testprefix fts5multiclient
+return_if_no_fts5
+
+foreach_detail_mode $testprefix {
+
+do_multiclient_test tn {
+
+  do_test 1.$tn.1 {
+    sql1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%) }
+    sql1 { INSERT INTO t1 VALUES('a b c') }
+    sql2 { SELECT rowid FROM t1('b') }
+  } {1}
+
+  do_test 1.$tn.2 {
+    sql2 { INSERT INTO t1 VALUES('a b c') }
+    sql1 { SELECT rowid FROM t1('b') }
+  } {1 2}
+
+  do_test 1.$tn.3 {
+    sql2 { INSERT INTO t1 VALUES('a b c') }
+    sql1 { SELECT rowid FROM t1('b') }
+  } {1 2 3}
+
+  do_test 1.$tn.4 {
+    sql2 { INSERT INTO t1 VALUES('a b c') }
+    sql1 { INSERT INTO t1 VALUES('a b c') }
+    sql3 { INSERT INTO t1(t1) VALUES('integrity-check') }
+  } {}
+
+};# do_multiclient_test
+};# foreach_detail_mode
+finish_test
@@ -68,4 +68,3 @@ do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0


 finish_test
-
@@ -178,4 +178,3 @@ do_execsql_test 4.3.1 {
 do_test 4.2.2 { fts5_level_segs ttt } {3}

 finish_test
-
@@ -20,6 +20,12 @@ ifcapable !fts5 {
  return
 }

+#
+# 1.* - Warm body tests for index optimization using ('optimize')
+#
+# 2.* - Warm body tests for index optimization using ('merge', -1)
+#
+
 proc rnddoc {nWord} {
  set vocab {a b c d e f g h i j k l m n o p q r s t u v w x y z}
  set nVocab [llength $vocab]
@@ -30,14 +36,12 @@ proc rnddoc {nWord} {
  return $ret
 }

-
 foreach {tn nStep} {
  1 2
  2 10
  3 50
  4 500
 } {
-if {$tn!=4} continue
  reset_db
  db func rnddoc rnddoc
  do_execsql_test 1.$tn.1 {
@@ -60,7 +64,45 @@ if {$tn!=4} continue
  do_execsql_test 1.$tn.5 {
    INSERT INTO t1(t1) VALUES('integrity-check');
  }
+
+  do_test 1.$tn.6 { fts5_segcount t1 } 1
 }

-finish_test
+foreach {tn nStep} {
+  1 2
+  2 10
+  3 50
+  4 500
+} {
+  reset_db
+  db func rnddoc rnddoc
+  do_execsql_test 1.$tn.1 {
+    CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+  }
+  do_test 2.$tn.2 {
+    for {set i 0} {$i < $nStep} {incr i} {
+      execsql { INSERT INTO t1 VALUES( rnddoc(5), rnddoc(5) ) }
+    }
+  } {}

+  do_execsql_test 2.$tn.3 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+
+  do_test 2.$tn.4 {
+    execsql { INSERT INTO t1(t1, rank) VALUES('merge', -1) }
+    while 1 {
+      set c [db total_changes]
+      execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
+      set c [expr [db total_changes]-$c]
+      if {$c<2} break
+    }
+  } {}
+
+  do_execsql_test 2.$tn.5 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+
+  do_test 2.$tn.6 { fts5_segcount t1 } 1
+}
+finish_test
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .11.0
 .20.0
				`@@ -66,4 +66,3 @@ do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }`


				`finish_test`
				`@@ -351,4 +351,3 @@ do_faultsim_test 9.1 -faults oom-* -prep {`


				`finish_test`
				`@@ -110,4 +110,3 @@ do_faultsim_test 3.2 -faults oom-* -prep {`


				`finish_test`
				`@@ -130,4 +130,3 @@ do_faultsim_test 3.3 -faults oom-t* -body {`


				`finish_test`
				`@@ -68,4 +68,3 @@ do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0`


				`finish_test`