Merge documentation changes from branch "begin-concurrent" into this branch.

FossilOrigin-Name: 41e742bd0f94894b096b30474fb548f58951756d1ea47d0afb717a54802888f4
Add new documentation file begin_concurrent.md.
2019-01-11 15:22:27 +00:00 · 2019-01-11 15:06:39 +00:00 · 2019-01-11 14:59:24 +00:00 · 2019-01-11 14:58:03 +00:00 · 2019-01-02 17:08:06 +00:00 · 2019-01-02 17:00:00 +00:00
407 changed files with 26338 additions and 4480 deletions
@@ -22,7 +22,7 @@ TOP = @abs_srcdir@
 #
 BCC = @BUILD_CC@ @BUILD_CFLAGS@

-# TCC is the C Compile and options for use in building executables that 
+# TCC is the C Compile and options for use in building executables that
 # will run on the target platform.  (BCC and TCC are usually the
 # same unless your are cross-compiling.)  Separate CC and CFLAGS macros
 # are provide so that these aspects of the build process can be changed
@@ -32,10 +32,11 @@ CC = @CC@
 CFLAGS = @CPPFLAGS@ @CFLAGS@
 TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
 TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
+TCC += -I${TOP}/ext/userauth

 # Define this for the autoconf-based build, so that the code knows it can
 # include the generated config.h
-# 
+#
 TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite

 # Define -DNDEBUG to compile without debugging (i.e., for production usage)
@@ -66,7 +67,7 @@ LIBREADLINE = @TARGET_READLINE_LIBS@
 TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@

 # Any target libraries which libsqlite must be linked against
-# 
+#
 TLIBS = @LIBS@ $(LIBS)

 # Flags controlling use of the in memory btree implementation
@@ -78,8 +79,8 @@ TLIBS = @LIBS@ $(LIBS)
 TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@

 # Enable/disable loadable extensions, and other optional features
-# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).  
-# The same set of OMIT and ENABLE flags should be passed to the 
+# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
+# The same set of OMIT and ENABLE flags should be passed to the
 # LEMON parser generator and the mkkeywordhash tool as well.
 OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@

@@ -126,8 +127,8 @@ SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
 # If gcov support was enabled by the configure script, add the appropriate
 # flags here.  It's not always as easy as just having the user add the right
 # CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
-# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.  
-# Supposedly GCC does the right thing if you use --coverage, but in 
+# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.
+# Supposedly GCC does the right thing if you use --coverage, but in
 # practice it still fails.  See:
 #
 # http://www.mail-archive.com/debian-gcc@lists.debian.org/msg26197.html
@@ -187,7 +188,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
         random.lo resolve.lo rowset.lo rtree.lo \
         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
         table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
-         update.lo upsert.lo util.lo vacuum.lo \
+         update.lo userauth.lo upsert.lo util.lo vacuum.lo \
         vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
         vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
         window.lo utf.lo vtab.lo
@@ -355,6 +356,9 @@ SRC += \
 SRC += \
  $(TOP)/ext/session/sqlite3session.c \
  $(TOP)/ext/session/sqlite3session.h
+SRC += \
+  $(TOP)/ext/userauth/userauth.c \
+  $(TOP)/ext/userauth/sqlite3userauth.h
 SRC += \
  $(TOP)/ext/rbu/sqlite3rbu.h \
  $(TOP)/ext/rbu/sqlite3rbu.c
@@ -425,7 +429,7 @@ TESTSRC = \
  $(TOP)/ext/fts3/fts3_term.c \
  $(TOP)/ext/fts3/fts3_test.c  \
  $(TOP)/ext/session/test_session.c \
-  $(TOP)/ext/rbu/test_rbu.c 
+  $(TOP)/ext/rbu/test_rbu.c

 # Statically linked extensions
 #
@@ -437,6 +441,7 @@ TESTSRC += \
  $(TOP)/ext/misc/closure.c \
  $(TOP)/ext/misc/csv.c \
  $(TOP)/ext/misc/eval.c \
+  $(TOP)/ext/misc/explain.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
@@ -454,7 +459,8 @@ TESTSRC += \
  $(TOP)/ext/misc/totype.c \
  $(TOP)/ext/misc/unionvtab.c \
  $(TOP)/ext/misc/wholenumber.c \
-  $(TOP)/ext/misc/zipfile.c
+  $(TOP)/ext/misc/zipfile.c \
+  $(TOP)/ext/userauth/userauth.c

 # Source code to the library files needed by the test fixture
 #
@@ -470,6 +476,7 @@ TESTSRC2 = \
  $(TOP)/src/dbstat.c \
  $(TOP)/src/expr.c \
  $(TOP)/src/func.c \
+  $(TOP)/src/global.c \
  $(TOP)/src/insert.c \
  $(TOP)/src/wal.c \
  $(TOP)/src/main.c \
@@ -506,7 +513,7 @@ TESTSRC2 = \
  $(TOP)/ext/fts3/fts3_write.c \
  $(TOP)/ext/async/sqlite3async.c \
  $(TOP)/ext/session/sqlite3session.c \
-  $(TOP)/ext/misc/stmt.c 
+  $(TOP)/ext/misc/stmt.c

 # Header files used by all library source files.
 #
@@ -559,6 +566,8 @@ EXTHDR += \
  $(TOP)/ext/icu/sqliteicu.h
 EXTHDR += \
  $(TOP)/ext/rtree/sqlite3rtree.h
+EXTHDR += \
+  $(TOP)/ext/userauth/sqlite3userauth.h

 # executables needed for testing
 #
@@ -578,7 +587,8 @@ FUZZDATA = \
  $(TOP)/test/fuzzdata3.db \
  $(TOP)/test/fuzzdata4.db \
  $(TOP)/test/fuzzdata5.db \
-  $(TOP)/test/fuzzdata6.db
+  $(TOP)/test/fuzzdata6.db \
+  $(TOP)/test/fuzzdata7.db

 # Standard options to testfixture
 #
@@ -595,13 +605,14 @@ SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
+SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE
 SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
-DBFUZZ_OPT = 
+DBFUZZ_OPT =

 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
@@ -658,11 +669,28 @@ ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.
             $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)

 sessionfuzz$(TEXE):	$(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h
-	$(CC) $(CFLAGS) -I. -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)
+	$(LTLINK) -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)

 dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)

+DBFUZZ2_OPTS = \
+  -DSQLITE_THREADSAFE=0 \
+  -DSQLITE_OMIT_LOAD_EXTENSION \
+  -DSQLITE_ENABLE_DESERIALIZE \
+  -DSQLITE_DEBUG \
+  -DSQLITE_ENABLE_DBSTAT_VTAB \
+  -DSQLITE_ENABLE_RTREE \
+  -DSQLITE_ENABLE_FTS4 \
+  -DSQLITE_EANBLE_FTS5
+
+dbfuzz2:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
+	clang-6.0 $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \
+		-fsanitize=fuzzer,undefined,address -o dbfuzz2 \
+		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c
+	mkdir -p dbfuzz2-dir
+	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
+
 mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
 	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
 		$(TLIBS) -rpath "$(libdir)"
@@ -1098,7 +1126,10 @@ fts3_write.lo:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
 rtree.lo:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c

-sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
+sqlite3session.lo:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c
+
+userauth.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c

 json1.lo:	$(TOP)/ext/misc/json1.c
@@ -1126,7 +1157,7 @@ FTS5_SRC = \
   $(TOP)/ext/fts5/fts5_varint.c \
   $(TOP)/ext/fts5/fts5_vocab.c  \

-fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
+fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon
 	cp $(TOP)/ext/fts5/fts5parse.y .
 	rm -f fts5parse.h
 	./lemon$(BEXE) $(OPTS) fts5parse.y
@@ -1153,12 +1184,13 @@ sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 #
 TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
 TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
-TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
+TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
+TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DESERIALIZE

 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
 TESTFIXTURE_SRC1 = sqlite3.c
@@ -1169,6 +1201,10 @@ testfixture$(TEXE):	$(TESTFIXTURE_SRC)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

+coretestprogs:	$(TESTPROGS)
+
+testprogs:	coretestprogs srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE)
+
 # A very detailed test running most or all test cases
 fulltest:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/all.test $(TESTOPTS)
@@ -1228,7 +1264,7 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)

-sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in	
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c

 sqltclsh$(TEXE): sqltclsh.c
@@ -1257,6 +1293,9 @@ dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
 	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
           $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)

+dbtotxt$(TEXE): $(TOP)/tool/dbtotxt.c
+	$(LTLINK)-o $@ $(TOP)/tool/dbtotxt.c
+
 showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)

@@ -1272,9 +1311,15 @@ showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
 showshm$(TEXE):	$(TOP)/tool/showshm.c
 	$(LTLINK) -o $@ $(TOP)/tool/showshm.c

+index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)
+
 changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)

+changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)
+
 rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)

@@ -1295,7 +1340,7 @@ KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
 kvtest$(TEXE):	$(TOP)/test/kvtest.c sqlite3.c
 	$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)

-rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
+rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo
 	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)

 loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
@@ -1323,7 +1368,7 @@ snapshot-tarball: sqlite3.c
 # The next two rules are used to support the "threadtest" target. Building
 # threadtest runs a few thread-safety tests that are implemented in C. This
 # target is invoked by the releasetest.tcl script.
-# 
+#
 THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
                  $(TOP)/test/tt3_checkpoint.c \
                  $(TOP)/test/tt3_index.c      \
@@ -1337,7 +1382,7 @@ threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC)
 threadtest: threadtest3$(TEXE)
 	./threadtest3$(TEXE)

-releasetest:	
+releasetest:
 	$(TCLSH_CMD) $(TOP)/test/releasetest.tcl

 # Standard install and cleanup targets
@@ -1345,7 +1390,7 @@ releasetest:
 lib_install:	libsqlite3.la
 	$(INSTALL) -d $(DESTDIR)$(libdir)
 	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
-	
+
 install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
 	$(INSTALL) -d $(DESTDIR)$(bindir)
 	$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
@@ -1363,7 +1408,7 @@ tcl_install:	lib_install libtclsqlite3.la pkgIndex.tcl
 	rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a
 	$(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)

-clean:	
+clean:
 	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
 	rm -f sqlite3.h opcodes.*
 	rm -rf .libs .deps
@@ -345,6 +345,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
@@ -1515,6 +1516,7 @@ TESTEXT = \
  $(TOP)\ext\misc\closure.c \
  $(TOP)\ext\misc\csv.c \
  $(TOP)\ext\misc\eval.c \
+  $(TOP)\ext\misc\explain.c \
  $(TOP)\ext\misc\fileio.c \
  $(TOP)\ext\misc\fuzzer.c \
  $(TOP)\ext\fts5\fts5_tcl.c \
@@ -1622,7 +1624,8 @@ FUZZDATA = \
  $(TOP)\test\fuzzdata3.db \
  $(TOP)\test\fuzzdata4.db \
  $(TOP)\test\fuzzdata5.db \
-  $(TOP)\test\fuzzdata6.db
+  $(TOP)\test\fuzzdata6.db \
+  $(TOP)\test\fuzzdata7.db
 # <</mark>>

 # Additional compiler options for the shell.  These are only effective
@@ -1632,6 +1635,7 @@ FUZZDATA = \
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF

 # <<mark>>
@@ -2294,6 +2298,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)

 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
@@ -2334,6 +2339,10 @@ extensiontest:	testfixture.exe testloadext.dll
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)

+coretestprogs:	$(TESTPROGS)
+
+testprogs:	coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe
+
 fulltest:	$(TESTPROGS) fuzztest
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\all.test $(TESTOPTS)
@@ -2417,6 +2426,9 @@ testloadext.lo:	$(TOP)\src\test_loadext.c $(SQLITE3H)
 testloadext.dll:	testloadext.lo
 	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo

+dbtotxt.exe:	$(TOP)\tool\dbtotxt.c
+	$(LTLINK) $(NO_WARN)	$(TOP)\tool\dbtotxt.c /link $(LDFLAGS) $(LTLINKOPTS)
+
 showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -2436,11 +2448,20 @@ showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
 showshm.exe:	$(TOP)\tool\showshm.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)

+index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		$(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
 		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

+changesetfuzz.exe:	$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
+		$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1 +1 @@
-3.25.0
+3.27.0
@@ -283,6 +283,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
@@ -937,6 +938,7 @@ LIBRESOBJS =
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF


@@ -172,7 +172,7 @@ AC_ARG_ENABLE(session, [AS_HELP_STRING(
  [--enable-session], [enable the session extension [default=no]])], 
  [], [])
 if test x"$enable_session" = "xyes"; then
-  BUILD_CFLAGS="$BUILD_CFLAGS-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
 fi
 #-----------------------------------------------------------------------

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.25.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.27.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.25.0'
-PACKAGE_STRING='sqlite 3.25.0'
+PACKAGE_VERSION='3.27.0'
+PACKAGE_STRING='sqlite 3.27.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -1466,7 +1466,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.25.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.27.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1531,7 +1531,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.25.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.27.0:";;
   esac
  cat <<\_ACEOF

@@ -1657,7 +1657,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.25.0
+sqlite configure 3.27.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2076,7 +2076,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.25.0, which was
+It was created by sqlite $as_me 3.27.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -12232,7 +12232,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.25.0, which was
+This file was extended by sqlite $as_me 3.27.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12298,7 +12298,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.25.0
+sqlite config.status 3.27.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -0,0 +1,107 @@
+
+Begin Concurrent
+================
+
+## Overview
+
+Usually, SQLite allows at most one writer to proceed concurrently. The
+BEGIN CONCURRENT enhancement allows multiple writers to process write
+transactions simultanously if the database is in "wal" or "wal2" mode,
+although the system still serializes COMMIT commands.
+
+When a write-transaction is opened with "BEGIN CONCURRENT", actually 
+locking the database is deferred until a COMMIT is executed. This means
+that any number of transactions started with BEGIN CONCURRENT may proceed
+concurrently. The system uses optimistic page-level-locking to prevent
+conflicting concurrent transactions from being committed.
+
+When a BEGIN CONCURRENT transaction is committed, the system checks whether 
+or not any of the database pages that the transaction has read have been
+modified since the BEGIN CONCURRENT was opened. In other words - it asks 
+if the transaction being committed operates on a different set of data than
+all other concurrently executing transactions. If the answer is "yes, this
+transaction did not read or modify any data modified by any concurrent
+transaction", then the transaction is committed as normal. Otherwise, if the
+transaction does conflict, it cannot be committed and an SQLITE&#95;BUSY&#95;SNAPSHOT
+error is returned. At this point, all the client can do is ROLLBACK the
+transaction.
+
+If SQLITE&#95;BUSY&#95;SNAPSHOT is returned, messages are output via the sqlite3&#95;log
+mechanism indicating the page and table or index on which the conflict
+occurred. This can be useful when optimizing concurrency.
+
+## Application Programming Notes
+
+In order to serialize COMMIT processing, SQLite takes a lock on the database
+as part of each COMMIT command and releases it before returning. At most one
+writer may hold this lock at any one time. If a writer cannot obtain the lock,
+it uses SQLite's busy-handler to pause and retry for a while:
+
+  <a href=https://www.sqlite.org/c3ref/busy_handler.html>
+      https://www.sqlite.org/c3ref/busy_handler.html
+  </a>
+
+If there is significant contention for the writer lock, this mechanism can be
+inefficient. In this case it is better for the application to use a mutex or
+some other mechanism that supports blocking to ensure that at most one writer
+is attempting to COMMIT a BEGIN CONCURRENT transaction at a time. This is
+usually easier if all writers are part of the same operating system process.
+
+If all database clients (readers and writers) are located in the same OS
+process, and if that OS is a Unix variant, then it can be more efficient to
+the built-in VFS "unix-excl" instead of the default "unix". This is because it
+uses more efficient locking primitives.
+
+The key to maximizing concurrency using BEGIN CONCURRENT is to ensure that
+there are a large number of non-conflicting transactions. In SQLite, each
+table and each index is stored as a separate b-tree, each of which is
+distributed over a discrete set of database pages. This means that:
+
+  * Two transactions that write to different sets of tables never 
+    conflict, and that
+
+  * Two transactions that write to the same tables or indexes only 
+    conflict if the values of the keys (either primary keys or indexed 
+    rows) are fairly close together. For example, given a large 
+    table with the schema:
+
+      <pre>     CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB);</pre>
+
+    writing two rows with adjacent values for "a" probably will cause a
+    conflict (as the two keys are stored on the same page), but writing two
+    rows with vastly different values for "a" will not (as the keys will likly
+    be stored on different pages).
+
+Note that, in SQLite, if values are not explicitly supplied for an INTEGER
+PRIMARY KEY, as for example in:
+
+>
+     INSERT INTO t1(b) VALUES(&lt;blob-value>);
+
+then monotonically increasing values are assigned automatically. This is
+terrible for concurrency, as it all but ensures that all new rows are 
+added to the same database page. In such situations, it is better to
+explicitly assign random values to INTEGER PRIMARY KEY fields.
+
+This problem also comes up for non-WITHOUT ROWID tables that do not have an
+explicit INTEGER PRIMARY KEY column. In these cases each table has an implicit
+INTEGER PRIMARY KEY column that is assigned increasing values, leading to the
+same problem as omitting to assign a value to an explicit INTEGER PRIMARY KEY
+column.
+
+For both explicit and implicit INTEGER PRIMARY KEYs, it is possible to have
+SQLite assign values at random (instead of the monotonically increasing
+values) by writing a row with a rowid equal to the largest possible signed
+64-bit integer to the table. For example:
+
+     INSERT INTO t1(a) VALUES(9223372036854775807);
+
+Applications should take care not to malfunction due to the presence of such
+rows.
+
+The nature of some types of indexes, for example indexes on timestamp fields,
+can also cause problems (as concurrent transactions may assign similar
+timestamps that will be stored on the same db page to new records). In these
+cases the database schema may need to be rethought to increase the concurrency
+provided by page-level-locking.
+
@@ -777,6 +777,9 @@ For example:</p>
 <p>This might be needed, for example, if some of the C actions in the
 grammar call functions that are prototyped in unistd.h.</p>

+<p>Use the <tt><a href="#pcode">%code</a></tt> directive to add code to
+the end of the generated parser.</p>
+
 <a name='pleft'></a>
 <h4>The <tt>%left</tt> directive</h4>

@@ -0,0 +1,98 @@
+
+Wal2 Mode Notes
+===============
+
+## Activating/Deactivating Wal2 Mode
+
+"Wal2" mode is very similar to "wal" mode. To change a database to wal2 mode,
+use the command:
+
+>
+     PRAGMA journal_mode = wal2;
+
+It is not possible to change a database directly from "wal" mode to "wal2"
+mode. Instead, it must first be changed to rollback mode. So, to change a wal
+mode database to wal2 mode, the following two commands may be used:
+
+>
+     PRAGMA journal_mode = delete;
+     PRAGMA journal_mode = wal2;
+
+A database in wal2 mode may only be accessed by versions of SQLite compiled
+from this branch. Attempting to use any other version of SQLite results in an
+SQLITE&#95;NOTADB error. A wal2 mode database may be changed back to rollback mode
+(making it accessible by all versions of SQLite) using:
+
+>
+     PRAGMA journal_mode = delete;
+
+## The Advantage of Wal2 Mode
+
+In legacy wal mode, when a writer writes data to the database, it doesn't
+modify the database file directly. Instead, it appends new data to the
+"&lt;database>-wal" file. Readers read data from both the original database
+file and the "&lt;database>-wal" file. At some point, data is copied from the
+"&lt;database>-wal" file into the database file, after which the wal file can
+be deleted or overwritten. Copying data from the wal file into the database
+file is called a "checkpoint", and may be done explictly (either by "PRAGMA
+wal&#95;checkpoint" or sqlite3&#95;wal&#95;checkpoint&#95;v2()), or
+automatically (by configuring "PRAGMA wal&#95;autocheckpoint" - this is the
+default).
+
+Checkpointers do not block writers, and writers do not block checkpointers.
+However, if a writer writes to the database while a checkpoint is ongoing,
+then the new data is appended to the end of the wal file. This means that,
+even following the checkpoint, the wal file cannot be overwritten or deleted,
+and so all subsequent transactions must also be appended to the wal file. The
+work of the checkpointer is not wasted - SQLite remembers which parts of the
+wal file have already been copied into the db file so that the next checkpoint
+does not have to do so again - but it does mean that the wal file may grow
+indefinitely if the checkpointer never gets a chance to finish without a
+writer appending to the wal file. There are also circumstances in which
+long-running readers may prevent a checkpointer from checkpointing the entire
+wal file - also causing the wal file to grow indefinitely in a busy system.
+
+Wal2 mode does not have this problem. In wal2 mode, wal files do not grow
+indefinitely even if the checkpointer never has a chance to finish
+uninterrupted.
+
+In wal2 mode, the system uses two wal files instead of one. The files are named
+"&lt;database>-wal" and "&lt;database>-wal2", where "&lt;database>" is of
+course the name of the database file. When data is written to the database, the
+writer begins by appending the new data to the first wal file. Once the first
+wal file has grown large enough, writers switch to appending data to the second
+wal file. At this point the first wal file can be checkpointed (after which it
+can be overwritten). Then, once the second wal file has grown large enough and
+the first wal file has been checkpointed, writers switch back to the first wal
+file. And so on.
+
+## Application Programming
+
+From the point of view of the user, the main differences between wal and 
+wal2 mode are to do with checkpointing:
+
+  * In wal mode, a checkpoint may be attempted at any time. In wal2 
+    mode, the checkpointer has to wait until writers have switched 
+    to the "other" wal file before a checkpoint can take place.
+
+  * In wal mode, the wal-hook (callback registered using
+    sqlite3&#95;wal&#95;hook()) is invoked after a transaction is committed
+    with the total number of pages in the wal file as an argument. In wal2
+    mode, the argument is either the total number of uncheckpointed pages in
+    both wal files, or - if the "other" wal file is empty or already
+    checkpointed - 0.
+
+Clients are recommended to use the same strategies for checkpointing wal2 mode
+databases as for wal databases - by registering a wal-hook using
+sqlite3&#95;wal&#95;hook() and attempting a checkpoint when the parameter
+exceeds a certain threshold.
+
+However, it should be noted that although the wal-hook is invoked after each
+transaction is committed to disk and database locks released, it is still
+invoked from within the sqlite3&#95;step() call used to execute the "COMMIT"
+command. In BEGIN CONCURRENT systems, where the "COMMIT" is often protected by
+an application mutex, this may reduce concurrency. In such systems, instead of
+executing a checkpoint from within the wal-hook, a thread might defer this
+action until after the application mutex has been released.
+
+
@@ -243,8 +243,10 @@ do_setup_rec_test $tn.12.1 {
  CREATE INDEX t7_idx_00000062 ON t7(b);
  CREATE INDEX t7_idx_00000061 ON t7(a);
  MULTI-INDEX OR
-    SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
-    SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
+    INDEX 1
+      SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
+    INDEX 2
+      SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }

 # rowid terms.
@@ -644,6 +644,7 @@ static int idxRegisterVtab(sqlite3expert *p){
    0,                            /* xSavepoint */
    0,                            /* xRelease */
    0,                            /* xRollbackTo */
+    0,                            /* xShadowName */
  };

  return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
@@ -338,7 +338,7 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
 }

 #define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \
-  v = (v & mask1) | ( (*ptr++) << shift );                    \
+  v = (v & mask1) | ( (*(ptr++)) << shift );  \
  if( (v & mask2)==0 ){ var = v; return ret; }
 #define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \
  v = (*ptr++);                                               \
@@ -376,20 +376,21 @@ int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
 ** a non-negative 32-bit integer before it is returned.
 */
 int sqlite3Fts3GetVarint32(const char *p, int *pi){
+  const unsigned char *ptr = (const unsigned char*)p;
  u32 a;

 #ifndef fts3GetVarint32
-  GETVARINT_INIT(a, p, 0,  0x00,     0x80, *pi, 1);
+  GETVARINT_INIT(a, ptr, 0,  0x00,     0x80, *pi, 1);
 #else
-  a = (*p++);
+  a = (*ptr++);
  assert( a & 0x80 );
 #endif

-  GETVARINT_STEP(a, p, 7,  0x7F,     0x4000, *pi, 2);
-  GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *pi, 3);
-  GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4);
+  GETVARINT_STEP(a, ptr, 7,  0x7F,     0x4000, *pi, 2);
+  GETVARINT_STEP(a, ptr, 14, 0x3FFF,   0x200000, *pi, 3);
+  GETVARINT_STEP(a, ptr, 21, 0x1FFFFF, 0x10000000, *pi, 4);
  a = (a & 0x0FFFFFFF );
-  *pi = (int)(a | ((u32)(*p & 0x07) << 28));
+  *pi = (int)(a | ((u32)(*ptr & 0x07) << 28));
  assert( 0==(a & 0x80000000) );
  assert( *pi>=0 );
  return 5;
@@ -1821,7 +1822,7 @@ static int fts3ScanInteriorNode(
  const char *zCsr = zNode;       /* Cursor to iterate through node */
  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
  char *zBuffer = 0;              /* Buffer to load terms into */
-  int nAlloc = 0;                 /* Size of allocated buffer */
+  i64 nAlloc = 0;                 /* Size of allocated buffer */
  int isFirstTerm = 1;            /* True when processing first term on page */
  sqlite3_int64 iChild;           /* Block id of child node to descend to */

@@ -1859,14 +1860,14 @@ static int fts3ScanInteriorNode(
    zCsr += fts3GetVarint32(zCsr, &nSuffix);
    
    assert( nPrefix>=0 && nSuffix>=0 );
-    if( &zCsr[nSuffix]>zEnd ){
+    if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
-    if( nPrefix+nSuffix>nAlloc ){
+    if( (i64)nPrefix+nSuffix>nAlloc ){
      char *zNew;
-      nAlloc = (nPrefix+nSuffix) * 2;
-      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+      nAlloc = ((i64)nPrefix+nSuffix) * 2;
+      zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
      if( !zNew ){
        rc = SQLITE_NOMEM;
        goto finish_scan;
@@ -3846,8 +3847,23 @@ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  return SQLITE_OK;
 }

+/*
+** Return true if zName is the extension on one of the shadow tables used
+** by this module.
+*/
+static int fts3ShadowName(const char *zName){
+  static const char *azName[] = {
+    "content", "docsize", "segdir", "segments", "stat", 
+  };
+  unsigned int i;
+  for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
+    if( sqlite3_stricmp(zName, azName[i])==0 ) return 1;
+  }
+  return 0;
+}
+
 static const sqlite3_module fts3Module = {
-  /* iVersion      */ 2,
+  /* iVersion      */ 3,
  /* xCreate       */ fts3CreateMethod,
  /* xConnect      */ fts3ConnectMethod,
  /* xBestIndex    */ fts3BestIndexMethod,
@@ -3870,6 +3886,7 @@ static const sqlite3_module fts3Module = {
  /* xSavepoint    */ fts3SavepointMethod,
  /* xRelease      */ fts3ReleaseMethod,
  /* xRollbackTo   */ fts3RollbackToMethod,
+  /* xShadowName   */ fts3ShadowName,
 };

 /*
@@ -4150,6 +4167,7 @@ static int fts3EvalPhraseLoad(
  return rc;
 }

+#ifndef SQLITE_DISABLE_FTS4_DEFERRED
 /*
 ** This function is called on each phrase after the position lists for
 ** any deferred tokens have been loaded into memory. It updates the phrases
@@ -4253,6 +4271,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){

  return SQLITE_OK;
 }
+#endif /* SQLITE_DISABLE_FTS4_DEFERRED */

 /*
 ** Maximum number of tokens a phrase may have to be considered for the
@@ -539,7 +539,8 @@ int sqlite3Fts3InitAux(sqlite3 *db){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
  };
  int rc;                         /* Return code */

@@ -361,7 +361,8 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
  };
  int rc;                         /* Return code */

@@ -443,7 +443,8 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
  };
  int rc;                         /* Return code */

@@ -82,7 +82,7 @@ typedef struct unicode_cursor unicode_cursor;

 struct unicode_tokenizer {
  sqlite3_tokenizer base;
-  int bRemoveDiacritic;
+  int eRemoveDiacritic;
  int nException;
  int *aiException;
 };
@@ -227,17 +227,20 @@ static int unicodeCreate(
  pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
  if( pNew==NULL ) return SQLITE_NOMEM;
  memset(pNew, 0, sizeof(unicode_tokenizer));
-  pNew->bRemoveDiacritic = 1;
+  pNew->eRemoveDiacritic = 1;

  for(i=0; rc==SQLITE_OK && i<nArg; i++){
    const char *z = azArg[i];
    int n = (int)strlen(z);

    if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 1;
+      pNew->eRemoveDiacritic = 1;
    }
    else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 0;
+      pNew->eRemoveDiacritic = 0;
+    }
+    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
+      pNew->eRemoveDiacritic = 2;
    }
    else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
      rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
@@ -350,7 +353,7 @@ static int unicodeNext(

    /* Write the folded case of the last character read to the output */
    zEnd = z;
-    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
+    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
    if( iOut ){
      WRITE_UTF8(zOut, iOut);
    }
@@ -159,32 +159,48 @@ int sqlite3FtsUnicodeIsalnum(int c){
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int remove_diacritic(int c){
+static int remove_diacritic(int c, int bComplex){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
  };
+#define HIBIT ((char)0x80)
  char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+    '\0',      'a',       'c',       'e',       'i',       'n',       
+    'o',       'u',       'y',       'y',       'a',       'c',       
+    'd',       'e',       'e',       'g',       'h',       'i',       
+    'j',       'k',       'l',       'n',       'o',       'r',       
+    's',       't',       'u',       'u',       'w',       'y',       
+    'z',       'o',       'u',       'a',       'i',       'o',       
+    'u',       'u'|HIBIT, 'a'|HIBIT, 'g',       'k',       'o',       
+    'o'|HIBIT, 'j',       'g',       'n',       'a'|HIBIT, 'a',       
+    'e',       'i',       'o',       'r',       'u',       's',       
+    't',       'h',       'a',       'e',       'o'|HIBIT, 'o',       
+    'o'|HIBIT, 'y',       '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a',       'b',       
+    'c'|HIBIT, 'd',       'd',       'e'|HIBIT, 'e',       'e'|HIBIT, 
+    'f',       'g',       'h',       'h',       'i',       'i'|HIBIT, 
+    'k',       'l',       'l'|HIBIT, 'l',       'm',       'n',       
+    'o'|HIBIT, 'p',       'r',       'r'|HIBIT, 'r',       's',       
+    's'|HIBIT, 't',       'u',       'u'|HIBIT, 'v',       'w',       
+    'w',       'x',       'y',       'z',       'h',       't',       
+    'w',       'y',       'a',       'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 
+    'e',       'e'|HIBIT, 'e'|HIBIT, 'i',       'o',       'o'|HIBIT, 
+    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',       
  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -201,7 +217,8 @@ static int remove_diacritic(int c){
    }
  }
  assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }


@@ -214,8 +231,8 @@ int sqlite3FtsUnicodeIsdiacritic(int c){
  unsigned int mask1 = 0x000361F8;
  if( c<768 || c>817 ) return 0;
  return (c < 768+32) ?
-      (mask0 & (1 << (c-768))) :
-      (mask1 & (1 << (c-768-32)));
+      (mask0 & ((unsigned int)1 << (c-768))) :
+      (mask1 & ((unsigned int)1 << (c-768-32)));
 }


@@ -228,7 +245,7 @@ int sqlite3FtsUnicodeIsdiacritic(int c){
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
@@ -351,7 +368,9 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
      assert( ret>0 );
    }

-    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = remove_diacritic(ret, eRemoveDiacritic==2);
+    }
  }
  
  else if( c>=66560 && c<66600 ){
@@ -396,10 +396,12 @@ static int fts3SqlStmt(
  
  pStmt = p->aStmt[eStmt];
  if( !pStmt ){
+    int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB;
    char *zSql;
    if( eStmt==SQL_CONTENT_INSERT ){
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
    }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
+      f &= ~SQLITE_PREPARE_NO_VTAB;
      zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
    }else{
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
@@ -407,8 +409,7 @@ static int fts3SqlStmt(
    if( !zSql ){
      rc = SQLITE_NOMEM;
    }else{
-      rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
-                              &pStmt, NULL);
+      rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL);
      sqlite3_free(zSql);
      assert( rc==SQLITE_OK || pStmt==0 );
      p->aStmt[eStmt] = pStmt;
@@ -1374,15 +1375,19 @@ static int fts3SegReaderNext(
  ** safe (no risk of overread) even if the node data is corrupted. */
  pNext += fts3GetVarint32(pNext, &nPrefix);
  pNext += fts3GetVarint32(pNext, &nSuffix);
-  if( nPrefix<0 || nSuffix<=0 
-   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
+  if( nSuffix<=0 
+   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
+   || nPrefix>pReader->nTermAlloc
  ){
    return FTS_CORRUPT_VTAB;
  }

-  if( nPrefix+nSuffix>pReader->nTermAlloc ){
-    int nNew = (nPrefix+nSuffix)*2;
-    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
+  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
+  ** overflow - hence the (i64) casts.  */
+  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
+    i64 nNew = ((i64)nPrefix+nSuffix)*2;
+    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
@@ -1404,7 +1409,7 @@ static int fts3SegReaderNext(
  ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  ** of these statements is untrue, then the data structure is corrupt.
  */
-  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
+  if( pReader->nDoclist > pReader->nNode-(pReader->aDoclist-pReader->aNode)
   || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  ){
    return FTS_CORRUPT_VTAB;
@@ -1604,7 +1609,11 @@ int sqlite3Fts3SegReaderNew(
  Fts3SegReader *pReader;         /* Newly allocated SegReader object */
  int nExtra = 0;                 /* Bytes to allocate segment root node */

-  assert( iStartLeaf<=iEndLeaf );
+  assert( zRoot!=0 || nRoot==0 );
+#ifdef CORRUPT_DB
+  assert( zRoot!=0 || CORRUPT_DB );
+#endif
+
  if( iStartLeaf==0 ){
    nExtra = nRoot + FTS3_NODE_PADDING;
  }
@@ -1625,7 +1634,7 @@ int sqlite3Fts3SegReaderNew(
    pReader->aNode = (char *)&pReader[1];
    pReader->rootOnly = 1;
    pReader->nNode = nRoot;
-    memcpy(pReader->aNode, zRoot, nRoot);
+    if( nRoot ) memcpy(pReader->aNode, zRoot, nRoot);
    memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING);
  }else{
    pReader->iCurrentBlock = iStartLeaf-1;
@@ -3730,6 +3739,9 @@ static int nodeReaderNext(NodeReader *p){
    }
    p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);

+    if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
+      return SQLITE_CORRUPT_VTAB;
+    }
    blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
    if( rc==SQLITE_OK ){
      memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
@@ -3737,6 +3749,9 @@ static int nodeReaderNext(NodeReader *p){
      p->iOff += nSuffix;
      if( p->iChild==0 ){
        p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
+        if( (p->nNode-p->iOff)<p->nDoclist ){
+          return SQLITE_CORRUPT_VTAB;
+        }
        p->aDoclist = &p->aNode[p->iOff];
        p->iOff += p->nDoclist;
      }
@@ -3744,7 +3759,6 @@ static int nodeReaderNext(NodeReader *p){
  }

  assert( p->iOff<=p->nNode );
-
  return rc;
 }

@@ -9,11 +9,12 @@ proc print_rd {map} {
  set nRange 1
  set iFirst  [lindex $map 0 0]
  set cPrev   [lindex $map 0 1]
+  set fPrev   [lindex $map 0 2]

  foreach m [lrange $map 1 end] {
-    foreach {i c} $m {}
+    foreach {i c f} $m {}

-    if {$cPrev == $c} {
+    if {$cPrev == $c && $fPrev==$f} {
      for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
        if {[info exists tl_lookup_table($j)]==0} break
      }
@@ -29,13 +30,16 @@ proc print_rd {map} {

    lappend lRange [list $iFirst $nRange]
    lappend aChar  $cPrev
+    lappend aFlag  $fPrev

    set iFirst $i
    set cPrev  $c
+    set fPrev  $f
    set nRange 1
  }
  lappend lRange [list $iFirst $nRange]
  lappend aChar $cPrev
+  lappend aFlag $fPrev

  puts "/*"
  puts "** If the argument is a codepoint corresponding to a lowercase letter"
@@ -45,7 +49,7 @@ proc print_rd {map} {
  puts "** E\"). The resuls of passing a codepoint that corresponds to an"
  puts "** uppercase letter are undefined."
  puts "*/"
-  puts "static int ${::remove_diacritic}(int c)\{"
+  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
  puts "  unsigned short aDia\[\] = \{"
  puts -nonewline "        0, "
  set i 1
@@ -59,14 +63,19 @@ proc print_rd {map} {
  }
  puts ""
  puts "  \};"
+  puts "#define HIBIT ((char)0x80)"
  puts "  char aChar\[\] = \{"
-  puts -nonewline "    '\\0', "
+  puts -nonewline "    '\\0',      "
  set i 1
-  foreach c $aChar {
-    set str "'$c',  "
-    if {$c == ""} { set str "'\\0', " }
+  foreach c $aChar f $aFlag {
+    if { $f } {
+      set str "'$c'|HIBIT, "
+    } else {
+      set str "'$c',       "
+    }
+    if {$c == ""} { set str "'\\0',      " }

-    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
+    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
    incr i
    puts -nonewline "$str"
  }
@@ -87,7 +96,8 @@ proc print_rd {map} {
    }
  }
  assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
  puts "\}"
 }

@@ -95,7 +105,8 @@ proc print_isdiacritic {zFunc map} {

  set lCode [list]
  foreach m $map {
-    foreach {code char} $m {}
+    foreach {code char flag} $m {}
+    if {$flag} continue
    if {$code && $char == ""} { lappend lCode $code }
  }
  set lCode [lsort -integer $lCode]
@@ -124,8 +135,8 @@ proc print_isdiacritic {zFunc map} {

  puts "  if( c<$iFirst || c>$iLast ) return 0;"
  puts "  return (c < $iFirst+32) ?"
-  puts "      (mask0 & (1 << (c-$iFirst))) :"
-  puts "      (mask1 & (1 << (c-$iFirst-32)));"
+  puts "      (mask0 & ((unsigned int)1 << (c-$iFirst))) :"
+  puts "      (mask1 & ((unsigned int)1 << (c-$iFirst-32)));"
  puts "\}"
 }

@@ -472,7 +483,7 @@ proc print_fold {zFunc} {
  puts "** The results are undefined if the value passed to this function"
  puts "** is less than zero."
  puts "*/"
-  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
+  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"

  set liOff [tl_generate_ioff_table $lRecord]
  tl_print_table_header
@@ -516,7 +527,9 @@ proc print_fold {zFunc} {
      assert( ret>0 );
    }

-    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
+    if( eRemoveDiacritic ){
+      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
+    }
  }
  }]

@@ -605,10 +618,6 @@ proc print_categories {lMap} {

  set nCat [expr [llength [array names C]] + 1]
  puts [code {
-    int sqlite3Fts5UnicodeNCat(void) { 
-      return $nCat;
-    }
-
    int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
      aArray[0] = 1;
      switch( zCat[0] ){
@@ -691,7 +700,7 @@ proc print_categories {lMap} {
    static u16 aFts5UnicodeMap[] = {$aMapArray};
    static u16 aFts5UnicodeData[] = {$aDataArray};

-    int sqlite3Fts5UnicodeCategory(int iCode) { 
+    int sqlite3Fts5UnicodeCategory(u32 iCode) { 
      int iRes = -1;
      int iHi;
      int iLo;
@@ -774,7 +783,7 @@ proc print_test_categories {lMap} {
          aArray[0] = 1;
        }

-        c = sqlite3Fts5UnicodeCategory(i);
+        c = sqlite3Fts5UnicodeCategory((u32)i);
        if( aArray[c]==0 ){
          *piCode = i;
          return 1;
@@ -7,12 +7,24 @@
 # character that it should be replaced with, or an empty string if the 
 # codepoint should simply be removed from the input. Examples:
 #
-#   { 224 a  }     (replace codepoint 224 to "a")
-#   { 769 "" }     (remove codepoint 769 from input)
+#   { 224 a  0 }     (replace codepoint 224 to "a")
+#   { 769 "" 0 }     (remove codepoint 769 from input)
 #
 # Mappings are only returned for non-upper case codepoints. It is assumed
 # that the input has already been folded to lower case.
 #
+# The third value in the list is always either 0 or 1. 0 if the 
+# UnicodeData.txt file maps the codepoint to a single ASCII character and
+# a diacritic, or 1 if the mapping is indirect. For example, consider the 
+# two entries:
+#
+# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
+# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
+#
+# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
+# diacritic). The second is an indirect mapping, as it maps to the
+# first codepoint plus 0302 (a diacritic).
+#
 proc rd_load_unicodedata_text {zName} {
  global tl_lookup_table

@@ -53,18 +65,29 @@ proc rd_load_unicodedata_text {zName} {
    set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
    set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]

+    # Filter out upper-case characters, as they will be mapped to their
+    # lower-case equivalents before this data is used.
    if {[info exists tl_lookup_table($iCode)]} continue

+    # Check if this is an indirect mapping. If so, set bIndirect to true
+    # and change $iAscii to the indirectly mappped ASCII character.
+    set bIndirect 0
+    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
+      set iAscii $mapping($iAscii)
+      set bIndirect 1
+    }
+
    if { ($iAscii >= 97 && $iAscii <= 122)
      || ($iAscii >= 65 && $iAscii <= 90)
    } {
-      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
+      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
+      set mapping($iCode) $iAscii
      set dia($iDia) 1
    }
  }

  foreach d [array names dia] {
-    lappend lRet [list $d ""]
+    lappend lRet [list $d "" 0]
  }
  set lRet [lsort -integer -index 0 $lRet]

@@ -414,11 +414,11 @@ struct Fts5ExtensionApi {
 **            the tokenizer substitutes "first" for "1st" and the query works
 **            as expected.
 **
-**       <li> By adding multiple synonyms for a single term to the FTS index.
-**            In this case, when tokenizing query text, the tokenizer may 
-**            provide multiple synonyms for a single term within the document.
-**            FTS5 then queries the index for each synonym individually. For
-**            example, faced with the query:
+**       <li> By querying the index for all synonyms of each query term
+**            separately. In this case, when tokenizing query text, the
+**            tokenizer may provide multiple synonyms for a single term 
+**            within the document. FTS5 then queries the index for each 
+**            synonym individually. For example, faced with the query:
 **
 **   <codeblock>
 **     ... MATCH 'first place'</codeblock>
@@ -442,7 +442,7 @@ struct Fts5ExtensionApi {
 **            "place".
 **
 **            This way, even if the tokenizer does not provide synonyms
-**            when tokenizing query text (it should not - to do would be
+**            when tokenizing query text (it should not - to do so would be
 **            inefficient), it doesn't matter if the user queries for 
 **            'first + place' or '1st + place', as there are entries in the
 **            FTS index corresponding to both forms of the first token.
@@ -788,7 +788,7 @@ int sqlite3Fts5UnicodeIsdiacritic(int c);
 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);

 int sqlite3Fts5UnicodeCatParse(const char*, u8*);
-int sqlite3Fts5UnicodeCategory(int iCode);
+int sqlite3Fts5UnicodeCategory(u32 iCode);
 void sqlite3Fts5UnicodeAscii(u8*, u8*);
 /*
 ** End of interface to code in fts5_unicode2.c.
@@ -2553,7 +2553,7 @@ static void fts5ExprIsAlnum(
  sqlite3Fts5UnicodeCatParse("N*", aArr);
  sqlite3Fts5UnicodeCatParse("Co", aArr);
  iCode = sqlite3_value_int(apVal[0]);
-  sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]);
+  sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory((u32)iCode)]);
 }

 static void fts5ExprFold(
@@ -690,6 +690,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
        pRet = 0;
      }else{
        /* TODO1: Fix this */
+        pRet->p[nByte] = 0x00;
        pRet->szLeaf = fts5GetU16(&pRet->p[2]);
      }
    }
@@ -729,7 +730,8 @@ static int fts5IndexPrepareStmt(
  if( p->rc==SQLITE_OK ){
    if( zSql ){
      p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1,
-                                 SQLITE_PREPARE_PERSISTENT, ppStmt, 0);
+          SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB,
+          ppStmt, 0);
    }else{
      p->rc = SQLITE_NOMEM;
    }
@@ -770,23 +772,12 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){
  if( p->rc!=SQLITE_OK ) return;

  if( p->pDeleter==0 ){
-    int rc;
    Fts5Config *pConfig = p->pConfig;
    char *zSql = sqlite3_mprintf(
        "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", 
          pConfig->zDb, pConfig->zName
    );
-    if( zSql==0 ){
-      rc = SQLITE_NOMEM;
-    }else{
-      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
-                              SQLITE_PREPARE_PERSISTENT, &p->pDeleter, 0);
-      sqlite3_free(zSql);
-    }
-    if( rc!=SQLITE_OK ){
-      p->rc = rc;
-      return;
-    }
+    if( fts5IndexPrepareStmt(p, &p->pDeleter, zSql) ) return;
  }

  sqlite3_bind_int64(p->pDeleter, 1, iFirst);
@@ -869,6 +860,11 @@ static int fts5StructureDecode(
  ** structure record.  */
  i += fts5GetVarint32(&pData[i], nLevel);
  i += fts5GetVarint32(&pData[i], nSegment);
+  if( nLevel>FTS5_MAX_SEGMENT   || nLevel<0
+   || nSegment>FTS5_MAX_SEGMENT || nSegment<0
+  ){
+    return FTS5_CORRUPT;
+  }
  nByte = (
      sizeof(Fts5Structure) +                    /* Main structure */
      sizeof(Fts5StructureLevel) * (nLevel-1)    /* aLevel[] array */
@@ -891,25 +887,35 @@ static int fts5StructureDecode(
      }else{
        i += fts5GetVarint32(&pData[i], pLvl->nMerge);
        i += fts5GetVarint32(&pData[i], nTotal);
-        assert( nTotal>=pLvl->nMerge );
+        if( nTotal<pLvl->nMerge ) rc = FTS5_CORRUPT;
        pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, 
            nTotal * sizeof(Fts5StructureSegment)
        );
+        nSegment -= nTotal;
      }

      if( rc==SQLITE_OK ){
        pLvl->nSeg = nTotal;
        for(iSeg=0; iSeg<nTotal; iSeg++){
+          Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
          if( i>=nData ){
            rc = FTS5_CORRUPT;
            break;
          }
-          i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid);
-          i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst);
-          i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast);
+          i += fts5GetVarint32(&pData[i], pSeg->iSegid);
+          i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst);
+          i += fts5GetVarint32(&pData[i], pSeg->pgnoLast);
+          if( pSeg->pgnoLast<pSeg->pgnoFirst ){
+            rc = FTS5_CORRUPT;
+            break;
+          }
        }
+        if( iLvl>0 && pLvl[-1].nMerge && nTotal==0 ) rc = FTS5_CORRUPT;
+        if( iLvl==nLevel-1 && pLvl->nMerge ) rc = FTS5_CORRUPT;
      }
    }
+    if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT;
+
    if( rc!=SQLITE_OK ){
      fts5StructureRelease(pRet);
      pRet = 0;
@@ -1647,12 +1653,13 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
  int nNew;                       /* Bytes of new data */

  iOff += fts5GetVarint32(&a[iOff], nNew);
-  if( iOff+nNew>pIter->pLeaf->nn ){
+  if( iOff+nNew>pIter->pLeaf->nn || nKeep>pIter->term.n ){
    p->rc = FTS5_CORRUPT;
    return;
  }
  pIter->term.n = nKeep;
  fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]);
+  assert( pIter->term.n<=pIter->term.nSpace );
  iOff += nNew;
  pIter->iTermLeafOffset = iOff;
  pIter->iTermLeafPgno = pIter->iLeafPgno;
@@ -1717,7 +1724,7 @@ static void fts5SegIterInit(
  if( p->rc==SQLITE_OK ){
    pIter->iLeafOffset = 4;
    assert_nc( pIter->pLeaf->nn>4 );
-    assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 );
+    assert_nc( fts5LeafFirstTermOff(pIter->pLeaf)==4 );
    pIter->iPgidxOff = pIter->pLeaf->szLeaf+1;
    fts5SegIterLoadTerm(p, pIter, 0);
    fts5SegIterLoadNPos(p, pIter);
@@ -2306,6 +2313,7 @@ static void fts5LeafSeek(
        iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff);
        if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){
          p->rc = FTS5_CORRUPT;
+          return;
        }else{
          nKeep = 0;
          iTermOff = iOff;
@@ -2318,8 +2326,11 @@ static void fts5LeafSeek(
  }

 search_success:
-
  pIter->iLeafOffset = iOff + nNew;
+  if( pIter->iLeafOffset>n ){
+    p->rc = FTS5_CORRUPT;
+    return;
+  }
  pIter->iTermLeafOffset = pIter->iLeafOffset;
  pIter->iTermLeafPgno = pIter->iLeafPgno;

@@ -3574,23 +3585,23 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
          int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid;
          if( iId<=FTS5_MAX_SEGMENT ){
-            aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32);
+            aUsed[(iId-1) / 32] |= (u32)1 << ((iId-1) % 32);
          }
        }
      }

      for(i=0; aUsed[i]==0xFFFFFFFF; i++);
      mask = aUsed[i];
-      for(iSegid=0; mask & (1 << iSegid); iSegid++);
+      for(iSegid=0; mask & ((u32)1 << iSegid); iSegid++);
      iSegid += 1 + i*32;

 #ifdef SQLITE_DEBUG
      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
-          assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid );
+          assert_nc( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid );
        }
      }
-      assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT );
+      assert_nc( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT );

      {
        sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p);
@@ -3598,7 +3609,7 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
          u8 aBlob[2] = {0xff, 0xff};
          sqlite3_bind_int(pIdxSelect, 1, iSegid);
          sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
-          assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
+          assert_nc( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
          p->rc = sqlite3_reset(pIdxSelect);
          sqlite3_bind_null(pIdxSelect, 2);
        }
@@ -3899,6 +3910,7 @@ static void fts5WriteAppendTerm(
  int nPrefix;                    /* Bytes of prefix compression for term */
  Fts5PageWriter *pPage = &pWriter->writer;
  Fts5Buffer *pPgidx = &pWriter->writer.pgidx;
+  int nMin = MIN(pPage->term.n, nTerm);

  assert( p->rc==SQLITE_OK );
  assert( pPage->buf.n>=4 );
@@ -3940,13 +3952,13 @@ static void fts5WriteAppendTerm(
      ** inefficient, but still correct.  */
      int n = nTerm;
      if( pPage->term.n ){
-        n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm);
+        n = 1 + fts5PrefixCompress(nMin, pPage->term.p, pTerm);
      }
      fts5WriteBtreeTerm(p, pWriter, n, pTerm);
      pPage = &pWriter->writer;
    }
  }else{
-    nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm);
+    nPrefix = fts5PrefixCompress(nMin, pPage->term.p, pTerm);
    fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix);
  }

@@ -5402,7 +5414,7 @@ int sqlite3Fts5IndexQuery(
      fts5CloseReader(p);
    }

-    *ppIter = &pRet->base;
+    *ppIter = (Fts5IndexIter*)pRet;
    sqlite3Fts5BufferFree(&buf);
  }
  return fts5IndexReturn(p);
@@ -5864,7 +5876,7 @@ static void fts5IndexIntegrityCheckSegment(

      iOff = fts5LeafFirstTermOff(pLeaf);
      iRowidOff = fts5LeafFirstRowidOff(pLeaf);
-      if( iRowidOff>=iOff ){
+      if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){
        p->rc = FTS5_CORRUPT;
      }else{
        iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
@@ -6279,8 +6291,7 @@ static void fts5DecodeFunction(
  nSpace = n + FTS5_DATA_ZERO_PADDING;
  a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace);
  if( a==0 ) goto decode_out;
-  memcpy(a, aBlob, n);
-
+  if( n>0 ) memcpy(a, aBlob, n);

  fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno);

@@ -2645,9 +2645,24 @@ static void fts5SourceIdFunc(
  sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT);
 }

+/*
+** Return true if zName is the extension on one of the shadow tables used
+** by this module.
+*/
+static int fts5ShadowName(const char *zName){
+  static const char *azName[] = {
+    "config", "content", "data", "docsize", "idx"
+  };
+  unsigned int i;
+  for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
+    if( sqlite3_stricmp(zName, azName[i])==0 ) return 1;
+  }
+  return 0;
+}
+
 static int fts5Init(sqlite3 *db){
  static const sqlite3_module fts5Mod = {
-    /* iVersion      */ 2,
+    /* iVersion      */ 3,
    /* xCreate       */ fts5CreateMethod,
    /* xConnect      */ fts5ConnectMethod,
    /* xBestIndex    */ fts5BestIndexMethod,
@@ -2670,6 +2685,7 @@ static int fts5Init(sqlite3 *db){
    /* xSavepoint    */ fts5SavepointMethod,
    /* xRelease      */ fts5ReleaseMethod,
    /* xRollbackTo   */ fts5RollbackToMethod,
+    /* xShadowName   */ fts5ShadowName
  };

  int rc;
@@ -136,8 +136,9 @@ static int fts5StorageGetStmt(
    if( zSql==0 ){
      rc = SQLITE_NOMEM;
    }else{
-      rc = sqlite3_prepare_v3(pC->db, zSql, -1,
-                              SQLITE_PREPARE_PERSISTENT, &p->aStmt[eStmt], 0);
+      int f = SQLITE_PREPARE_PERSISTENT;
+      if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB;
+      rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
      sqlite3_free(zSql);
      if( rc!=SQLITE_OK && pzErrMsg ){
        *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
@@ -378,7 +378,7 @@ static int fts5tokFilterMethod(
    if( pCsr->zInput==0 ){
      rc = SQLITE_NOMEM;
    }else{
-      memcpy(pCsr->zInput, zByte, nByte);
+      if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte);
      pCsr->zInput[nByte] = 0;
      rc = pTab->tok.xTokenize(
          pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb
@@ -471,7 +471,8 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
  };
  int rc;                         /* Return code */

@@ -234,13 +234,18 @@ struct Unicode61Tokenizer {
  unsigned char aTokenChar[128];  /* ASCII range token characters */
  char *aFold;                    /* Buffer to fold text into */
  int nFold;                      /* Size of aFold[] in bytes */
-  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
+  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
  int nException;
  int *aiException;

  unsigned char aCategory[32];    /* True for token char categories */
 };

+/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
+#define FTS5_REMOVE_DIACRITICS_NONE    0
+#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
+#define FTS5_REMOVE_DIACRITICS_COMPLEX 2
+
 static int fts5UnicodeAddExceptions(
  Unicode61Tokenizer *p,          /* Tokenizer object */
  const char *z,                  /* Characters to treat as exceptions */
@@ -257,7 +262,7 @@ static int fts5UnicodeAddExceptions(
      const unsigned char *zCsr = (const unsigned char*)z;
      const unsigned char *zTerm = (const unsigned char*)&z[n];
      while( zCsr<zTerm ){
-        int iCode;
+        u32 iCode;
        int bToken;
        READ_UTF8(zCsr, zTerm, iCode);
        if( iCode<128 ){
@@ -269,7 +274,7 @@ static int fts5UnicodeAddExceptions(
          if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
            int i;
            for(i=0; i<nNew; i++){
-              if( aNew[i]>iCode ) break;
+              if( (u32)aNew[i]>iCode ) break;
            }
            memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int));
            aNew[i] = iCode;
@@ -361,7 +366,7 @@ static int fts5UnicodeCreate(
      int i;
      memset(p, 0, sizeof(Unicode61Tokenizer));

-      p->bRemoveDiacritic = 1;
+      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
      p->nFold = 64;
      p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
      if( p->aFold==0 ){
@@ -382,10 +387,15 @@ static int fts5UnicodeCreate(
      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
        const char *zArg = azArg[i+1];
        if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
+          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
            rc = SQLITE_ERROR;
+          }else{
+            p->eRemoveDiacritic = (zArg[0] - '0');
+            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
+                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
+                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
+            );
          }
-          p->bRemoveDiacritic = (zArg[0]=='1');
        }else
        if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
          rc = fts5UnicodeAddExceptions(p, zArg, 1);
@@ -419,7 +429,7 @@ static int fts5UnicodeCreate(
 */
 static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
  return (
-    p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]
+    p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)]
    ^ fts5UnicodeIsException(p, iCode)
  );
 }
@@ -448,7 +458,7 @@ static int fts5UnicodeTokenize(
  /* Each iteration of this loop gobbles up a contiguous run of separators,
  ** then the next token.  */
  while( rc==SQLITE_OK ){
-    int iCode;                    /* non-ASCII codepoint read from input */
+    u32 iCode;                    /* non-ASCII codepoint read from input */
    char *zOut = aFold;
    int is;
    int ie;
@@ -499,7 +509,7 @@ static int fts5UnicodeTokenize(
        READ_UTF8(zCsr, zTerm, iCode);
        if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
 non_ascii_tokenchar:
-          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
+          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
          if( iCode ) WRITE_UTF8(zOut, iCode);
        }else{
          break;
@@ -1274,5 +1284,3 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){

  return rc;
 }
-
-
@@ -28,32 +28,48 @@
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int fts5_remove_diacritic(int c){
+static int fts5_remove_diacritic(int c, int bComplex){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
  };
+#define HIBIT ((char)0x80)
  char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+    '\0',      'a',       'c',       'e',       'i',       'n',       
+    'o',       'u',       'y',       'y',       'a',       'c',       
+    'd',       'e',       'e',       'g',       'h',       'i',       
+    'j',       'k',       'l',       'n',       'o',       'r',       
+    's',       't',       'u',       'u',       'w',       'y',       
+    'z',       'o',       'u',       'a',       'i',       'o',       
+    'u',       'u'|HIBIT, 'a'|HIBIT, 'g',       'k',       'o',       
+    'o'|HIBIT, 'j',       'g',       'n',       'a'|HIBIT, 'a',       
+    'e',       'i',       'o',       'r',       'u',       's',       
+    't',       'h',       'a',       'e',       'o'|HIBIT, 'o',       
+    'o'|HIBIT, 'y',       '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a',       'b',       
+    'c'|HIBIT, 'd',       'd',       'e'|HIBIT, 'e',       'e'|HIBIT, 
+    'f',       'g',       'h',       'h',       'i',       'i'|HIBIT, 
+    'k',       'l',       'l'|HIBIT, 'l',       'm',       'n',       
+    'o'|HIBIT, 'p',       'r',       'r'|HIBIT, 'r',       's',       
+    's'|HIBIT, 't',       'u',       'u'|HIBIT, 'v',       'w',       
+    'w',       'x',       'y',       'z',       'h',       't',       
+    'w',       'y',       'a',       'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 
+    'e',       'e'|HIBIT, 'e'|HIBIT, 'i',       'o',       'o'|HIBIT, 
+    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',       
  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -70,7 +86,8 @@ static int fts5_remove_diacritic(int c){
    }
  }
  assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }


@@ -83,8 +100,8 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){
  unsigned int mask1 = 0x000361F8;
  if( c<768 || c>817 ) return 0;
  return (c < 768+32) ?
-      (mask0 & (1 << (c-768))) :
-      (mask1 & (1 << (c-768-32)));
+      (mask0 & ((unsigned int)1 << (c-768))) :
+      (mask1 & ((unsigned int)1 << (c-768-32)));
 }


@@ -97,7 +114,7 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
@@ -220,7 +237,9 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
      assert( ret>0 );
    }

-    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
+    }
  }
  
  else if( c>=66560 && c<66600 ){
@@ -231,12 +250,6 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
 }


-#if 0
-int sqlite3Fts5UnicodeNCat(void) { 
-  return 32;
-}
-#endif
-
 int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
  aArray[0] = 1;
  switch( zCat[0] ){
@@ -718,7 +731,7 @@ static u16 aFts5UnicodeData[] = {
    34,    3074,  7692,  63,    63,    
  };

-int sqlite3Fts5UnicodeCategory(int iCode) { 
+int sqlite3Fts5UnicodeCategory(u32 iCode) { 
  int iRes = -1;
  int iHi;
  int iLo;
@@ -756,7 +769,7 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
    int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
    int n = (aFts5UnicodeData[iTbl] >> 5) + i;
    for(; i<128 && i<n; i++){
-      aAscii[i] = (u8)bToken;
+      aAscii[i] = bToken;
    }
    iTbl++;
  }
@@ -431,6 +431,8 @@ static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
  i64 *pp = &pCsr->iInstPos;
  int *po = &pCsr->iInstOff;
  
+  assert( sqlite3Fts5IterEof(pIter)==0 );
+  assert( pCsr->bEof==0 );
  while( eDetail==FTS5_DETAIL_NONE
      || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) 
  ){
@@ -440,7 +442,7 @@ static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
    rc = sqlite3Fts5IterNextScan(pCsr->pIter);
    if( rc==SQLITE_OK ){
      rc = fts5VocabInstanceNewTerm(pCsr);
-      if( eDetail==FTS5_DETAIL_NONE ) break;
+      if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE ) break;
    }
    if( rc ){
      pCsr->bEof = 1;
@@ -755,10 +757,9 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
    /* xSavepoint    */ 0,
    /* xRelease      */ 0,
    /* xRollbackTo   */ 0,
+    /* xShadowName   */ 0
  };
  void *p = (void*)pGlobal;

  return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
 }
-
-
@@ -38,8 +38,7 @@ do_execsql_test 1.0 {
 do_execsql_test 1.1 {
  DROP TABLE t1;
  SELECT name, sql FROM sqlite_master;
-} {
-}
+} {}

 #-------------------------------------------------------------------------
 #
@@ -409,6 +408,7 @@ do_test 14.3 {
 do_execsql_test 15.0 {
  INSERT INTO t1(t1) VALUES('integrity-check');
 }
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 15.1 {
  UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1;
 }
@@ -0,0 +1,80 @@
+# 2018 Dec 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5circref
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE tt USING fts5(a);
+  SELECT name FROM sqlite_master ORDER BY 1;
+} {
+  tt tt_config tt_content tt_data tt_docsize tt_idx
+}
+db_save_and_close
+
+foreach {tn schema sql} {
+  1 {
+    CREATE TRIGGER tr1 AFTER INSERT ON tt_config BEGIN
+      SELECT * FROM tt;
+    END;
+  } {
+    INSERT INTO tt(tt, rank) VALUES('usermerge', 4);
+  }
+
+  2 {
+    CREATE TRIGGER tr1 AFTER INSERT ON tt_docsize BEGIN
+      SELECT * FROM tt;
+    END;
+  } {
+    INSERT INTO tt(a) VALUES('one two three');
+  }
+
+  3 {
+    CREATE TRIGGER tr1 AFTER INSERT ON tt_content BEGIN
+      SELECT * FROM tt;
+    END;
+  } {
+    INSERT INTO tt(a) VALUES('one two three');
+  }
+
+  4 {
+    CREATE TRIGGER tr1 AFTER INSERT ON tt_data BEGIN
+      SELECT * FROM tt;
+    END;
+  } {
+    INSERT INTO tt(a) VALUES('one two three');
+  }
+
+  5 {
+    CREATE TRIGGER tr1 AFTER INSERT ON tt_idx BEGIN
+      SELECT * FROM tt;
+    END;
+  } {
+    INSERT INTO tt(a) VALUES('one two three');
+  }
+} {
+  db_restore_and_reopen
+  do_execsql_test 1.1.$tn.1 $schema
+  do_catchsql_test 1.1.$tn.2 $sql {1 {SQL logic error}}
+  db close
+}
+
+
+finish_test
@@ -244,4 +244,3 @@ foreach {tn sql res} {
 }

 finish_test
-
@@ -41,6 +41,7 @@ db_save
 do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 set segid [lindex [fts5_level_segids t1] 0]

+sqlite3_db_config db DEFENSIVE 0
 do_test 1.3 {
  execsql {
    DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4);
@@ -50,6 +51,7 @@ do_test 1.3 {

 do_test 1.4 {
  db_restore_and_reopen
+  sqlite3_db_config db DEFENSIVE 0
  execsql {
    UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE
    rowid = fts5_rowid('segment', $segid, 4);
@@ -89,7 +91,7 @@ do_execsql_test 3.0 {
 do_execsql_test 3.1 {
  SELECT * FROM t3 WHERE t3 MATCH 'o'
 } {{one o} {three o} {five o}}
-
+sqlite3_db_config db DEFENSIVE 0
 do_catchsql_test 3.1 {
  DELETE FROM t3_content WHERE rowid = 3;
  SELECT * FROM t3 WHERE t3 MATCH 'o';
@@ -99,6 +99,7 @@ foreach {tno stmt} {
 set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] 
 set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
 set all [db eval {SELECT rowid FROM t1}]
+sqlite3_db_config db DEFENSIVE 0
 for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
  do_execsql_test 2.$i.1 {
    BEGIN;
@@ -248,6 +249,7 @@ foreach {tn hdr} {

 #--------------------------------------------------------------------
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE x5 USING fts5(tt);
  INSERT INTO x5 VALUES('a');
@@ -93,4 +93,3 @@ do_catchsql_test 3.3 {
  SELECT * FROM x2('^a');
 } {1 {fts5: phrase queries are not supported (detail!=full)}}
 finish_test
-
@@ -71,6 +71,7 @@ do_execsql_test 4.1 {
  INSERT INTO aa(aa) VALUES('integrity-check'); 
 }

+sqlite3_db_config db DEFENSIVE 0
 do_catchsql_test 4.2 { 
  BEGIN;
    UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;
@@ -163,4 +163,3 @@ do_execsql_test 5.1 {
 } {30 31 32 33 34 35 36 37 38 39 40}

 finish_test
-
@@ -39,6 +39,7 @@ do_execsql_test 1.4 {
  INSERT INTO f1(f1) VALUES('integrity-check');
 } {}

+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 1.5 {
  DELETE FROM f1_data;
 } {}
@@ -70,6 +70,7 @@ set res [db one {SELECT count(*) FROM x1_data}]
 do_execsql_test 2.3 {
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
 } $res
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 2.4 {
  UPDATE x1_data SET block = X'';
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
@@ -189,7 +189,7 @@ do_catchsql_test 6.2 {
 } {1 {error in tokenizer constructor}}
 do_catchsql_test 6.3 {
  CREATE VIRTUAL TABLE a3 USING fts5(
-    x, y, tokenize = 'unicode61 remove_diacritics 2'
+    x, y, tokenize = 'unicode61 remove_diacritics 3'
  );
 } {1 {error in tokenizer constructor}}
 do_catchsql_test 6.4 {
@@ -0,0 +1,65 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5umlaut
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      x, 
+      tokenize="unicode61 remove_diacritics 2"
+  );
+}
+
+foreach {tn q res1 res2} {
+  1 "Hà Nội"                  0 1
+  2 "Hà Noi"                  1 1
+  3 "Ha Noi"                  1 1
+  4 "Ha N\u1ed9i"             0 1
+  5 "Ha N\u006fi"             1 1
+  6 "Ha N\u006f\u0302i"       1 1
+  7 "Ha N\u006f\u0323\u0302i" 1 1
+} {
+  do_execsql_test 1.$tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t1($q)
+  } $res1
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t1('Ha Noi')
+  } $res1
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t2($q)
+  } $res2
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t2('Ha Noi')
+  } $res2
+}
+
+finish_test
+
@@ -21,7 +21,7 @@ ifcapable !fts5 {
 }

 proc fts3_unicode_path {file} {
-  file join [file dirname [info script]] .. .. fts3 unicode $file
+  file join .. [file dirname [info script]] .. .. fts3 unicode $file
 }

 source [fts3_unicode_path parseunicode.tcl]
@@ -36,24 +36,26 @@ foreach x [an_load_unicodedata_text $UD] {
 }

 foreach {y} [rd_load_unicodedata_text $UD] {
-  foreach {code ascii} $y {}
+  foreach {code ascii f} $y {}
  if {$ascii==""} {
    set int 0
  } else {
    binary scan $ascii c int
  }
-  set aDiacritic($code) $int
+  set aDiacritic($code,$f) $int
+  if {$f==0} { set aDiacritic($code,1) $int }
 }

 proc tcl_fold {i {bRemoveDiacritic 0}} {
  global tl_lookup_table
  global aDiacritic
+  set f [expr $bRemoveDiacritic==2]

  if {[info exists tl_lookup_table($i)]} {
    set i $tl_lookup_table($i)
  }
-  if {$bRemoveDiacritic && [info exists aDiacritic($i)]} {
-    set i $aDiacritic($i)
+  if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} {
+    set i $aDiacritic($i,$f)
  }
  expr $i
 }
@@ -85,7 +87,7 @@ do_execsql_test 1.1 {
  SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
 } {0 {}}

-do_execsql_test 1.2 {
+do_execsql_test 1.2.1 {
  WITH ii(i) AS (
    SELECT -1
    UNION ALL
@@ -95,6 +97,16 @@ do_execsql_test 1.2 {
  WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
 } {0 {}}

+do_execsql_test 1.2.2 {
+  WITH ii(i) AS (
+    SELECT -1
+    UNION ALL
+    SELECT i+1 FROM ii WHERE i<100000
+  )
+  SELECT count(*), min(i) FROM ii 
+  WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int);
+} {0 {}}
+
 do_execsql_test 1.3 {
  WITH ii(i) AS (
    SELECT -1
@@ -36,9 +36,10 @@ do_execsql_test 1.3 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'a';
 } {1}

+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 1.4 {
  UPDATE t1_config set v=5 WHERE k='version';
-} 
+}

 do_test 1.5 {
  db close
@@ -53,6 +54,7 @@ do_test 1.6 {
 } {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}

 do_test 1.7 {
+  sqlite3_db_config db DEFENSIVE 0
  execsql { DELETE FROM t1_config WHERE k='version' }
  db close
  sqlite3 db test.db
@@ -79,8 +79,8 @@ do_execsql_test 1.1.2 {
  3 cnt {} 0 {} 0
 }

-do_execsql_test 1.2.1 { SELECT * FROM v1 } { }
-do_execsql_test 1.2.2 { SELECT * FROM v2 } { }
+do_execsql_test 1.2.1 { SELECT * FROM v1 } {}
+do_execsql_test 1.2.2 { SELECT * FROM v2 } {}

 do_execsql_test 1.3 {
  INSERT INTO t1 VALUES('x y z');
@@ -420,6 +420,7 @@ if {[detail_is_none]} { set resc [row_to_col $resr] }
 do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
 do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc

+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 8.2 {
  PRAGMA writable_schema = 1;
  UPDATE sqlite_master 
@@ -481,4 +482,3 @@ do_test 9.6 {


 finish_test
-
@@ -13,7 +13,7 @@
 #

 source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5vocab
+set testprefix fts5vocab2

 # If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
@@ -80,8 +80,7 @@ do_execsql_test 1.4 {
 do_execsql_test 1.5 {
  DELETE FROM t1;
  SELECT * FROM v1;
-} {
-}
+} {}

 #-------------------------------------------------------------------------
 #
@@ -143,8 +142,7 @@ do_execsql_test 2.4 {
 do_execsql_test 2.5 {
  DELETE FROM t1;
  SELECT * FROM v1;
-} {
-}
+} {}

 #-------------------------------------------------------------------------
 #
@@ -202,8 +200,6 @@ do_execsql_test 3.4 {
 do_execsql_test 3.5 {
  DELETE FROM t1;
  SELECT * FROM v1;
-} {
-}
+} {}

 finish_test
-
@@ -1473,7 +1473,8 @@ static sqlite3_module amatchModule = {
  0,                      /* xRename */
  0,                      /* xSavepoint */
  0,                      /* xRelease */
-  0                       /* xRollbackTo */
+  0,                      /* xRollbackTo */
+  0                       /* xShadowName */
 };

 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -0,0 +1,244 @@
+/*
+** 2017-10-11
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+
+#if !defined(SQLITE_TEST) || defined(SQLITE_OS_UNIX)
+
+#include "sqlite3.h"
+#include <string.h>
+#include <pthread.h>
+
+/*
+** API declarations.
+*/
+typedef struct Checkpointer Checkpointer;
+int sqlite3_bgckpt_create(const char *zFilename, Checkpointer **pp);
+int sqlite3_bgckpt_checkpoint(Checkpointer *p, int bBlock);
+void sqlite3_bgckpt_destroy(Checkpointer *p);
+
+
+struct Checkpointer {
+  sqlite3 *db;                    /* Database handle */
+
+  pthread_t thread;               /* Background thread */
+  pthread_mutex_t mutex;
+  pthread_cond_t cond;
+
+  int rc;                         /* Error from "PRAGMA wal_checkpoint" */
+  int bCkpt;                      /* True if checkpoint requested */
+  int bExit;                      /* True if exit requested */
+};
+
+static void *bgckptThreadMain(void *pCtx){
+  int rc = SQLITE_OK;
+  Checkpointer *p = (Checkpointer*)pCtx;
+
+  while( rc==SQLITE_OK ){
+    int bExit;
+
+    pthread_mutex_lock(&p->mutex);
+    if( p->bCkpt==0 && p->bExit==0 ){
+      pthread_cond_wait(&p->cond, &p->mutex);
+    }
+    p->bCkpt = 0;
+    bExit = p->bExit;
+    pthread_mutex_unlock(&p->mutex);
+
+    if( bExit ) break;
+    rc = sqlite3_exec(p->db, "PRAGMA wal_checkpoint", 0, 0, 0);
+    if( rc==SQLITE_BUSY ){
+      rc = SQLITE_OK;
+    }
+  }
+
+  pthread_mutex_lock(&p->mutex);
+  p->rc = rc;
+  pthread_mutex_unlock(&p->mutex);
+  return 0;
+}
+
+void sqlite3_bgckpt_destroy(Checkpointer *p){
+  if( p ){
+    void *ret = 0;
+
+    /* Signal the background thread to exit */
+    pthread_mutex_lock(&p->mutex);
+    p->bExit = 1;
+    pthread_cond_broadcast(&p->cond);
+    pthread_mutex_unlock(&p->mutex);
+
+    pthread_join(p->thread, &ret);
+    sqlite3_close(p->db);
+    sqlite3_free(p);
+  }
+}
+
+
+int sqlite3_bgckpt_create(const char *zFilename, Checkpointer **pp){
+  Checkpointer *pNew = 0;
+  int rc;
+
+  pNew = (Checkpointer*)sqlite3_malloc(sizeof(Checkpointer));
+  if( pNew==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    memset(pNew, 0, sizeof(Checkpointer));
+    rc = sqlite3_open(zFilename, &pNew->db);
+  }
+
+  if( rc==SQLITE_OK ){
+    pthread_mutex_init(&pNew->mutex, 0);
+    pthread_cond_init(&pNew->cond, 0);
+    pthread_create(&pNew->thread, 0, bgckptThreadMain, (void*)pNew);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_bgckpt_destroy(pNew);
+    pNew = 0;
+  }
+  *pp = pNew;
+  return rc;
+}
+
+int sqlite3_bgckpt_checkpoint(Checkpointer *p, int bBlock){
+  int rc;
+  pthread_mutex_lock(&p->mutex);
+  rc = p->rc;
+  if( rc==SQLITE_OK ){
+    p->bCkpt = 1;
+    pthread_cond_broadcast(&p->cond);
+  }
+  pthread_mutex_unlock(&p->mutex);
+  return rc;
+}
+
+#ifdef SQLITE_TEST
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+
+const char *sqlite3ErrName(int rc);
+
+static void SQLITE_TCLAPI bgckpt_del(void * clientData){
+  Checkpointer *pCkpt = (Checkpointer*)clientData;
+  sqlite3_bgckpt_destroy(pCkpt);
+}
+
+/*
+** Tclcmd: $ckpt SUBCMD ...
+*/
+static int SQLITE_TCLAPI bgckpt_obj_cmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  Checkpointer *pCkpt = (Checkpointer*)clientData;
+  const char *aCmd[] = { "checkpoint", "destroy", 0 };
+  int iCmd;
+
+  if( objc<2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SUBCMD ...");
+    return TCL_ERROR;
+  }
+
+  if( Tcl_GetIndexFromObj(interp, objv[1], aCmd, "sub-command", 0, &iCmd) ){
+    return TCL_ERROR;
+  }
+
+  switch( iCmd ){
+    case 0: {
+      int rc;
+      int bBlock = 0;
+
+      if( objc>3 ){
+        Tcl_WrongNumArgs(interp, 2, objv, "?BLOCKING?");
+        return TCL_ERROR;
+      }
+      if( objc==3 && Tcl_GetBooleanFromObj(interp, objv[2], &bBlock) ){
+        return TCL_ERROR;
+      }
+
+      rc = sqlite3_bgckpt_checkpoint(pCkpt, bBlock);
+      if( rc!=SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+        return TCL_ERROR;
+      }
+      break;
+    }
+
+    case 1: {
+      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+      break;
+    }
+  }
+
+  return TCL_OK;
+}
+
+/*
+** Tclcmd: bgckpt CMDNAME FILENAME
+*/
+static int SQLITE_TCLAPI bgckpt_cmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  const char *zCmd;
+  const char *zFilename;
+  int rc;
+  Checkpointer *pCkpt;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "CMDNAME FILENAME");
+    return TCL_ERROR;
+  }
+  zCmd = Tcl_GetString(objv[1]);
+  zFilename = Tcl_GetString(objv[2]);
+
+  rc = sqlite3_bgckpt_create(zFilename, &pCkpt);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    return TCL_ERROR;
+  }
+
+  Tcl_CreateObjCommand(interp, zCmd, bgckpt_obj_cmd, (void*)pCkpt, bgckpt_del);
+  Tcl_SetObjResult(interp, objv[1]);
+  return TCL_OK;
+}
+
+int Bgckpt_Init(Tcl_Interp *interp){
+  Tcl_CreateObjCommand(interp, "bgckpt", bgckpt_cmd, 0, 0);
+  return TCL_OK;
+}
+#endif   /* SQLITE_TEST */
+
+#else
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+int Bgckpt_Init(Tcl_Interp *interp){ return TCL_OK; }
+#endif
+
@@ -411,6 +411,7 @@ int sqlite3BinfoRegister(sqlite3 *db){
    0,                           /* xSavepoint */
    0,                           /* xRelease */
    0,                           /* xRollbackTo */
+    0                            /* xShadowName */
  };
  return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0);
 }
@@ -938,7 +938,8 @@ static sqlite3_module closureModule = {
  0,                      /* xRename */
  0,                      /* xSavepoint */
  0,                      /* xRelease */
-  0                       /* xRollbackTo */
+  0,                      /* xRollbackTo */
+  0                       /* xShadowName */
 };

 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -468,7 +468,8 @@ static sqlite3_module completionModule = {
  0,                         /* xRename */
  0,                         /* xSavepoint */
  0,                         /* xRelease */
-  0                          /* xRollbackTo */
+  0,                         /* xRollbackTo */
+  0                          /* xShadowName */
 };

 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -19,9 +19,9 @@
 **    CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
 **    SELECT * FROM csv;
 **
-** The columns are named "c1", "c2", "c3", ... by default.  But the
-** application can define its own CREATE TABLE statement as an additional
-** parameter.  For example:
+** The columns are named "c1", "c2", "c3", ... by default.  Or the
+** application can define its own CREATE TABLE statement using the
+** schema= parameter, like this:
 **
 **    CREATE VIRTUAL TABLE temp.csv2 USING csv(
 **       filename = "../http.log",
@@ -32,9 +32,9 @@
 ** the data= parameter.
 **
 ** If the columns=N parameter is supplied, then the CSV file is assumed to have
-** N columns.  If the columns parameter is omitted, the CSV file is opened
-** as soon as the virtual table is constructed and the first row of the CSV
-** is read in order to count the tables.
+** N columns.  If both the columns= and schema= parameters are omitted, then
+** the number and names of the columns is determined by the first line of
+** the CSV input.
 **
 ** Some extra debugging features (used for testing virtual tables) are available
 ** if this module is compiled with -DSQLITE_TEST.
@@ -436,6 +436,34 @@ static int csv_boolean(const char *z){
  return -1;
 }

+/* Check to see if the string is of the form:  "TAG = BOOLEAN" or just "TAG".
+** If it is, set *pValue to be the value of the boolean ("true" if there is
+** not "= BOOLEAN" component) and return non-zero.  If the input string
+** does not begin with TAG, return zero.
+*/
+static int csv_boolean_parameter(
+  const char *zTag,       /* Tag we are looking for */
+  int nTag,               /* Size of the tag in bytes */
+  const char *z,          /* Input parameter */
+  int *pValue             /* Write boolean value here */
+){
+  int b;
+  z = csv_skip_whitespace(z);
+  if( strncmp(zTag, z, nTag)!=0 ) return 0;
+  z = csv_skip_whitespace(z + nTag);
+  if( z[0]==0 ){
+    *pValue = 1;
+    return 1;
+  }
+  if( z[0]!='=' ) return 0;
+  z = csv_skip_whitespace(z+1);
+  b = csv_boolean(z);
+  if( b>=0 ){
+    *pValue = b;
+    return 1;
+  }
+  return 0;
+}

 /*
 ** Parameters:
@@ -469,6 +497,7 @@ static int csvtabConnect(
 #ifdef SQLITE_TEST
  int tstFlags = 0;          /* Value for testflags=N parameter */
 #endif
+  int b;                     /* Value of a boolean parameter */
  int nCol = -99;            /* Value of the columns= parameter */
  CsvReader sRdr;            /* A CSV file reader used to store an error
                             ** message and/or to count the number of columns */
@@ -493,21 +522,12 @@ static int csvtabConnect(
    if( j<sizeof(azParam)/sizeof(azParam[0]) ){
      if( sRdr.zErr[0] ) goto csvtab_connect_error;
    }else
-    if( (zValue = csv_parameter("header",6,z))!=0 ){
-      int x;
+    if( csv_boolean_parameter("header",6,z,&b) ){
      if( bHeader>=0 ){
        csv_errmsg(&sRdr, "more than one 'header' parameter");
        goto csvtab_connect_error;
      }
-      x = csv_boolean(zValue);
-      if( x==1 ){
-        bHeader = 1;
-      }else if( x==0 ){
-        bHeader = 0;
-      }else{
-        csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue);
-        goto csvtab_connect_error;
-      }
+      bHeader = b;
    }else
 #ifdef SQLITE_TEST
    if( (zValue = csv_parameter("testflags",9,z))!=0 ){
@@ -521,53 +541,94 @@ static int csvtabConnect(
      }
      nCol = atoi(zValue);
      if( nCol<=0 ){
-        csv_errmsg(&sRdr, "must have at least one column");
+        csv_errmsg(&sRdr, "column= value must be positive");
        goto csvtab_connect_error;
      }
    }else
    {
-      csv_errmsg(&sRdr, "unrecognized parameter '%s'", z);
+      csv_errmsg(&sRdr, "bad parameter: '%s'", z);
      goto csvtab_connect_error;
    }
  }
  if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
-    csv_errmsg(&sRdr, "must either filename= or data= but not both");
+    csv_errmsg(&sRdr, "must specify either filename= or data= but not both");
    goto csvtab_connect_error;
  }
-  if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){
+
+  if( (nCol<=0 || bHeader==1)
+   && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA)
+  ){
    goto csvtab_connect_error;
  }
  pNew = sqlite3_malloc( sizeof(*pNew) );
  *ppVtab = (sqlite3_vtab*)pNew;
  if( pNew==0 ) goto csvtab_connect_oom;
  memset(pNew, 0, sizeof(*pNew));
-  if( nCol>0 ){
+  if( CSV_SCHEMA==0 ){
+    sqlite3_str *pStr = sqlite3_str_new(0);
+    char *zSep = "";
+    int iCol = 0;
+    sqlite3_str_appendf(pStr, "CREATE TABLE x(");
+    if( nCol<0 && bHeader<1 ){
+      nCol = 0;
+      do{
+        csv_read_one_field(&sRdr);
+        nCol++;
+      }while( sRdr.cTerm==',' );
+    }
+    if( nCol>0 && bHeader<1 ){
+      for(iCol=0; iCol<nCol; iCol++){
+        sqlite3_str_appendf(pStr, "%sc%d TEXT", zSep, iCol);
+        zSep = ",";
+      }
+    }else{
+      do{
+        char *z = csv_read_one_field(&sRdr);
+        if( (nCol>0 && iCol<nCol) || (nCol<0 && bHeader) ){
+          sqlite3_str_appendf(pStr,"%s\"%w\" TEXT", zSep, z);
+          zSep = ",";
+          iCol++;
+        }
+      }while( sRdr.cTerm==',' );
+      if( nCol<0 ){
+        nCol = iCol;
+      }else{
+        while( iCol<nCol ){
+          sqlite3_str_appendf(pStr,"%sc%d TEXT", zSep, ++iCol);
+          zSep = ",";
+        }
+      }
+    }
    pNew->nCol = nCol;
-  }else{
+    sqlite3_str_appendf(pStr, ")");
+    CSV_SCHEMA = sqlite3_str_finish(pStr);
+    if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
+  }else if( nCol<0 ){
    do{
      csv_read_one_field(&sRdr);
      pNew->nCol++;
    }while( sRdr.cTerm==',' );
+  }else{
+    pNew->nCol = nCol;
  }
  pNew->zFilename = CSV_FILENAME;  CSV_FILENAME = 0;
  pNew->zData = CSV_DATA;          CSV_DATA = 0;
 #ifdef SQLITE_TEST
  pNew->tstFlags = tstFlags;
 #endif
-  pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0;
-  csv_reader_reset(&sRdr);
-  if( CSV_SCHEMA==0 ){
-    char *zSep = "";
-    CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x(");
-    if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
-    for(i=0; i<pNew->nCol; i++){
-      CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i);
-      zSep = ",";
-    }
-    CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA);
+  if( bHeader!=1 ){
+    pNew->iStart = 0;
+  }else if( pNew->zData ){
+    pNew->iStart = (int)sRdr.iIn;
+  }else{
+    pNew->iStart = ftell(sRdr.in) - sRdr.nIn + sRdr.iIn;
  }
+  csv_reader_reset(&sRdr);
  rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
-  if( rc ) goto csvtab_connect_error;
+  if( rc ){
+    csv_errmsg(&sRdr, "bad schema: '%s' - %s", CSV_SCHEMA, sqlite3_errmsg(db));
+    goto csvtab_connect_error;
+  }
  for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
    sqlite3_free(azPValue[i]);
  }
@@ -0,0 +1,322 @@
+/*
+** 2018-09-16
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file demonstrates an eponymous virtual table that returns the
+** EXPLAIN output from an SQL statement.
+**
+** Usage example:
+**
+**     .load ./explain
+**     SELECT p2 FROM explain('SELECT * FROM sqlite_master')
+**      WHERE opcode='OpenRead';
+**
+** This module was originally written to help simplify SQLite testing,
+** by providing an easier means of verifying certain patterns in the
+** generated bytecode.
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* explain_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a explain virtual table
+*/
+typedef struct explain_vtab explain_vtab;
+struct explain_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  sqlite3 *db;        /* Database connection for this explain vtab */
+};
+
+/* explain_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result from an EXPLAIN operation.
+*/
+typedef struct explain_cursor explain_cursor;
+struct explain_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  sqlite3 *db;               /* Database connection for this cursor */
+  char *zSql;                /* Value for the EXPLN_COLUMN_SQL column */
+  sqlite3_stmt *pExplain;    /* Statement being explained */
+  int rc;                    /* Result of last sqlite3_step() on pExplain */
+};
+
+/*
+** The explainConnect() method is invoked to create a new
+** explain_vtab that describes the explain virtual table.
+**
+** Think of this routine as the constructor for explain_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the explain_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against explain will look like.
+*/
+static int explainConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  explain_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define EXPLN_COLUMN_ADDR     0   /* Instruction address */
+#define EXPLN_COLUMN_OPCODE   1   /* Opcode */
+#define EXPLN_COLUMN_P1       2   /* Operand 1 */
+#define EXPLN_COLUMN_P2       3   /* Operand 2 */
+#define EXPLN_COLUMN_P3       4   /* Operand 3 */
+#define EXPLN_COLUMN_P4       5   /* Operand 4 */
+#define EXPLN_COLUMN_P5       6   /* Operand 5 */
+#define EXPLN_COLUMN_COMMENT  7   /* Comment */
+#define EXPLN_COLUMN_SQL      8   /* SQL that is being explained */
+
+
+  rc = sqlite3_declare_vtab(db,
+     "CREATE TABLE x(addr,opcode,p1,p2,p3,p4,p5,comment,sql HIDDEN)");
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->db = db;
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for explain_cursor objects.
+*/
+static int explainDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new explain_cursor object.
+*/
+static int explainOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  explain_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->db = ((explain_vtab*)p)->db;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a explain_cursor.
+*/
+static int explainClose(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  sqlite3_finalize(pCur->pExplain);
+  sqlite3_free(pCur->zSql);
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a explain_cursor to its next row of output.
+*/
+static int explainNext(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  pCur->rc = sqlite3_step(pCur->pExplain);
+  if( pCur->rc!=SQLITE_DONE && pCur->rc!=SQLITE_ROW ) return pCur->rc;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the explain_cursor
+** is currently pointing.
+*/
+static int explainColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  if( i==EXPLN_COLUMN_SQL ){
+    sqlite3_result_text(ctx, pCur->zSql, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_value(ctx, sqlite3_column_value(pCur->pExplain, i));
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int explainRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  *pRowid = sqlite3_column_int64(pCur->pExplain, 0);
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int explainEof(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  return pCur->rc!=SQLITE_ROW;
+}
+
+/*
+** This method is called to "rewind" the explain_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to explainColumn() or explainRowid() or 
+** explainEof().
+**
+** The argv[0] is the SQL statement that is to be explained.
+*/
+static int explainFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  explain_cursor *pCur = (explain_cursor *)pVtabCursor;
+  char *zSql = 0;
+  int rc;
+  sqlite3_finalize(pCur->pExplain);
+  pCur->pExplain = 0;
+  if( sqlite3_value_type(argv[0])!=SQLITE_TEXT ){
+    pCur->rc = SQLITE_DONE;
+    return SQLITE_OK;
+  }
+  sqlite3_free(pCur->zSql);
+  pCur->zSql = sqlite3_mprintf("%s", sqlite3_value_text(argv[0]));
+  if( pCur->zSql ){
+    zSql = sqlite3_mprintf("EXPLAIN %s", pCur->zSql);
+  }
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pExplain, 0);
+    sqlite3_free(zSql);
+  }
+  if( rc ){
+    sqlite3_finalize(pCur->pExplain);
+    pCur->pExplain = 0;
+    sqlite3_free(pCur->zSql);
+    pCur->zSql = 0;
+  }else{
+    pCur->rc = sqlite3_step(pCur->pExplain);
+    rc = (pCur->rc==SQLITE_DONE || pCur->rc==SQLITE_ROW) ? SQLITE_OK : pCur->rc;
+  }
+  return rc;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the explain virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int explainBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;                  /* Loop counter */
+  int idx = -1;           /* Index of a usable == constraint against SQL */
+  int unusable = 0;       /* True if there are unusable constraints on SQL */
+
+  pIdxInfo->estimatedRows = 500;
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+    if( p->iColumn!=EXPLN_COLUMN_SQL ) continue;
+    if( !p->usable ){
+      unusable = 1;
+    }else if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      idx = i;
+    }
+  }
+  if( idx>=0 ){
+    /* There exists a usable == constraint against the SQL column */
+    pIdxInfo->estimatedCost = 10.0;
+    pIdxInfo->idxNum = 1;
+    pIdxInfo->aConstraintUsage[idx].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[idx].omit = 1;
+  }else if( unusable ){
+    /* There are unusable constraints against the SQL column.  Do not allow
+    ** this plan to continue forward. */
+    return SQLITE_CONSTRAINT;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** explain virtual table.
+*/
+static sqlite3_module explainModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  explainConnect,            /* xConnect */
+  explainBestIndex,          /* xBestIndex */
+  explainDisconnect,         /* xDisconnect */
+  0,                         /* xDestroy */
+  explainOpen,               /* xOpen - open a cursor */
+  explainClose,              /* xClose - close a cursor */
+  explainFilter,             /* xFilter - configure scan constraints */
+  explainNext,               /* xNext - advance a cursor */
+  explainEof,                /* xEof - check for end of scan */
+  explainColumn,             /* xColumn - read data */
+  explainRowid,              /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+int sqlite3ExplainVtabInit(sqlite3 *db){
+  int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3_create_module(db, "explain", &explainModule, 0);
+#endif
+  return rc;
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_explain_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3ExplainVtabInit(db);
+#endif
+  return rc;
+}
@@ -106,7 +106,18 @@ SQLITE_EXTENSION_INIT1
 #include <errno.h>


+/*
+** Structure of the fsdir() table-valued function
+*/
+                 /*    0    1    2     3    4           5             */
 #define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
+#define FSDIR_COLUMN_NAME     0     /* Name of the file */
+#define FSDIR_COLUMN_MODE     1     /* Access mode */
+#define FSDIR_COLUMN_MTIME    2     /* Last modification time */
+#define FSDIR_COLUMN_DATA     3     /* File content */
+#define FSDIR_COLUMN_PATH     4     /* Path to top of search */
+#define FSDIR_COLUMN_DIR      5     /* Path is relative to this directory */
+

 /*
 ** Set the result stored by context ctx to a blob containing the 
@@ -695,20 +706,20 @@ static int fsdirColumn(
 ){
  fsdir_cursor *pCur = (fsdir_cursor*)cur;
  switch( i ){
-    case 0: { /* name */
+    case FSDIR_COLUMN_NAME: {
      sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT);
      break;
    }

-    case 1: /* mode */
+    case FSDIR_COLUMN_MODE:
      sqlite3_result_int64(ctx, pCur->sStat.st_mode);
      break;

-    case 2: /* mtime */
+    case FSDIR_COLUMN_MTIME:
      sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
      break;

-    case 3: { /* data */
+    case FSDIR_COLUMN_DATA: {
      mode_t m = pCur->sStat.st_mode;
      if( S_ISDIR(m) ){
        sqlite3_result_null(ctx);
@@ -738,6 +749,12 @@ static int fsdirColumn(
        readFileContents(ctx, pCur->zPath);
      }
    }
+    case FSDIR_COLUMN_PATH:
+    default: {
+      /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters.
+      ** always return their values as NULL */
+      break;
+    }
  }
  return SQLITE_OK;
 }
@@ -764,6 +781,9 @@ static int fsdirEof(sqlite3_vtab_cursor *cur){

 /*
 ** xFilter callback.
+**
+** idxNum==1   PATH parameter only
+** idxNum==2   Both PATH and DIR supplied
 */
 static int fsdirFilter(
  sqlite3_vtab_cursor *cur, 
@@ -816,40 +836,63 @@ static int fsdirFilter(
 ** In this implementation idxNum is used to represent the
 ** query plan.  idxStr is unused.
 **
-** The query plan is represented by bits in idxNum:
+** The query plan is represented by values of idxNum:
 **
-**  (1)  start = $value  -- constraint exists
-**  (2)  stop = $value   -- constraint exists
-**  (4)  step = $value   -- constraint exists
-**  (8)  output in descending order
+**  (1)  The path value is supplied by argv[0]
+**  (2)  Path is in argv[0] and dir is in argv[1]
 */
 static int fsdirBestIndex(
  sqlite3_vtab *tab,
  sqlite3_index_info *pIdxInfo
 ){
  int i;                 /* Loop over constraints */
-  int idx4 = -1;
-  int idx5 = -1;
+  int idxPath = -1;      /* Index in pIdxInfo->aConstraint of PATH= */
+  int idxDir = -1;       /* Index in pIdxInfo->aConstraint of DIR= */
+  int seenPath = 0;      /* True if an unusable PATH= constraint is seen */
+  int seenDir = 0;       /* True if an unusable DIR= constraint is seen */
  const struct sqlite3_index_constraint *pConstraint;

  (void)tab;
  pConstraint = pIdxInfo->aConstraint;
  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->usable==0 ) continue;
    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    if( pConstraint->iColumn==4 ) idx4 = i;
-    if( pConstraint->iColumn==5 ) idx5 = i;
+    switch( pConstraint->iColumn ){
+      case FSDIR_COLUMN_PATH: {
+        if( pConstraint->usable ){
+          idxPath = i;
+          seenPath = 0;
+        }else if( idxPath<0 ){
+          seenPath = 1;
+        }
+        break;
+      }
+      case FSDIR_COLUMN_DIR: {
+        if( pConstraint->usable ){
+          idxDir = i;
+          seenDir = 0;
+        }else if( idxDir<0 ){
+          seenDir = 1;
+        }
+        break;
+      }
+    } 
+  }
+  if( seenPath || seenDir ){
+    /* If input parameters are unusable, disallow this plan */
+    return SQLITE_CONSTRAINT;
  }

-  if( idx4<0 ){
+  if( idxPath<0 ){
    pIdxInfo->idxNum = 0;
-    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
+    /* The pIdxInfo->estimatedCost should have been initialized to a huge
+    ** number.  Leave it unchanged. */
+    pIdxInfo->estimatedRows = 0x7fffffff;
  }else{
-    pIdxInfo->aConstraintUsage[idx4].omit = 1;
-    pIdxInfo->aConstraintUsage[idx4].argvIndex = 1;
-    if( idx5>=0 ){
-      pIdxInfo->aConstraintUsage[idx5].omit = 1;
-      pIdxInfo->aConstraintUsage[idx5].argvIndex = 2;
+    pIdxInfo->aConstraintUsage[idxPath].omit = 1;
+    pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1;
+    if( idxDir>=0 ){
+      pIdxInfo->aConstraintUsage[idxDir].omit = 1;
+      pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2;
      pIdxInfo->idxNum = 2;
      pIdxInfo->estimatedCost = 10.0;
    }else{
@@ -888,7 +931,8 @@ static int fsdirRegister(sqlite3 *db){
    0,                         /* xRename */
    0,                         /* xSavepoint */
    0,                         /* xRelease */
-    0                          /* xRollbackTo */
+    0,                         /* xRollbackTo */
+    0,                         /* xShadowName */
  };

  int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
@@ -1994,6 +1994,9 @@ static int jsonEachConnect(
 #define JEACH_PARENT  5
 #define JEACH_FULLKEY 6
 #define JEACH_PATH    7
+/* The xBestIndex method assumes that the JSON and ROOT columns are
+** the last two columns in the table.  Should this ever changes, be
+** sure to update the xBestIndex method. */
 #define JEACH_JSON    8
 #define JEACH_ROOT    9

@@ -2251,35 +2254,54 @@ static int jsonEachBestIndex(
  sqlite3_vtab *tab,
  sqlite3_index_info *pIdxInfo
 ){
-  int i;
-  int jsonIdx = -1;
-  int rootIdx = -1;
+  int i;                     /* Loop counter or computed array index */
+  int aIdx[2];               /* Index of constraints for JSON and ROOT */
+  int unusableMask = 0;      /* Mask of unusable JSON and ROOT constraints */
+  int idxMask = 0;           /* Mask of usable == constraints JSON and ROOT */
  const struct sqlite3_index_constraint *pConstraint;

+  /* This implementation assumes that JSON and ROOT are the last two
+  ** columns in the table */
+  assert( JEACH_ROOT == JEACH_JSON+1 );
  UNUSED_PARAM(tab);
+  aIdx[0] = aIdx[1] = -1;
  pConstraint = pIdxInfo->aConstraint;
  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->usable==0 ) continue;
-    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    switch( pConstraint->iColumn ){
-      case JEACH_JSON:   jsonIdx = i;    break;
-      case JEACH_ROOT:   rootIdx = i;    break;
-      default:           /* no-op */     break;
+    int iCol;
+    int iMask;
+    if( pConstraint->iColumn < JEACH_JSON ) continue;
+    iCol = pConstraint->iColumn - JEACH_JSON;
+    assert( iCol==0 || iCol==1 );
+    iMask = 1 << iCol;
+    if( pConstraint->usable==0 ){
+      unusableMask |= iMask;
+    }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      aIdx[iCol] = i;
+      idxMask |= iMask;
    }
  }
-  if( jsonIdx<0 ){
+  if( (unusableMask & ~idxMask)!=0 ){
+    /* If there are any unusable constraints on JSON or ROOT, then reject
+    ** this entire plan */
+    return SQLITE_CONSTRAINT;
+  }
+  if( aIdx[0]<0 ){
+    /* No JSON input.  Leave estimatedCost at the huge value that it was
+    ** initialized to to discourage the query planner from selecting this
+    ** plan. */
    pIdxInfo->idxNum = 0;
-    pIdxInfo->estimatedCost = 1e99;
  }else{
    pIdxInfo->estimatedCost = 1.0;
-    pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1;
-    pIdxInfo->aConstraintUsage[jsonIdx].omit = 1;
-    if( rootIdx<0 ){
-      pIdxInfo->idxNum = 1;
+    i = aIdx[0];
+    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[i].omit = 1;
+    if( aIdx[1]<0 ){
+      pIdxInfo->idxNum = 1;  /* Only JSON supplied.  Plan 1 */
    }else{
-      pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2;
-      pIdxInfo->aConstraintUsage[rootIdx].omit = 1;
-      pIdxInfo->idxNum = 3;
+      i = aIdx[1];
+      pIdxInfo->aConstraintUsage[i].argvIndex = 2;
+      pIdxInfo->aConstraintUsage[i].omit = 1;
+      pIdxInfo->idxNum = 3;  /* Both JSON and ROOT are supplied.  Plan 3 */
    }
  }
  return SQLITE_OK;
@@ -2388,7 +2410,8 @@ static sqlite3_module jsonEachModule = {
  0,                         /* xRename */
  0,                         /* xSavepoint */
  0,                         /* xRelease */
-  0                          /* xRollbackTo */
+  0,                         /* xRollbackTo */
+  0                          /* xShadowName */
 };

 /* The methods of the json_tree virtual table. */
@@ -2415,7 +2438,8 @@ static sqlite3_module jsonTreeModule = {
  0,                         /* xRename */
  0,                         /* xSavepoint */
  0,                         /* xRelease */
-  0                          /* xRollbackTo */
+  0,                         /* xRollbackTo */
+  0                          /* xShadowName */
 };
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

@@ -0,0 +1,428 @@
+/*
+** 2018-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file demonstrates an eponymous virtual table that returns information
+** from sqlite3_status64() and sqlite3_db_status().
+**
+** Usage example:
+**
+**     .load ./memstat
+**     .mode quote
+**     .header on
+**     SELECT * FROM memstat;
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB)
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* memstat_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a memstat virtual table
+*/
+typedef struct memstat_vtab memstat_vtab;
+struct memstat_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  sqlite3 *db;        /* Database connection for this memstat vtab */
+};
+
+/* memstat_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct memstat_cursor memstat_cursor;
+struct memstat_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  sqlite3 *db;               /* Database connection for this cursor */
+  int iRowid;                /* Current row in aMemstatColumn[] */
+  int iDb;                   /* Which schema we are looking at */
+  int nDb;                   /* Number of schemas */
+  char **azDb;               /* Names of all schemas */
+  sqlite3_int64 aVal[2];     /* Result values */
+};
+
+/*
+** The memstatConnect() method is invoked to create a new
+** memstat_vtab that describes the memstat virtual table.
+**
+** Think of this routine as the constructor for memstat_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the memstat_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against memstat will look like.
+*/
+static int memstatConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  memstat_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define MSV_COLUMN_NAME    0   /* Name of quantity being measured */
+#define MSV_COLUMN_SCHEMA  1   /* schema name */
+#define MSV_COLUMN_VALUE   2   /* Current value */
+#define MSV_COLUMN_HIWTR   3   /* Highwater mark */
+
+  rc = sqlite3_declare_vtab(db,"CREATE TABLE x(name,schema,value,hiwtr)");
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->db = db;
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for memstat_cursor objects.
+*/
+static int memstatDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new memstat_cursor object.
+*/
+static int memstatOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  memstat_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->db = ((memstat_vtab*)p)->db;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Clear all the schema names from a cursor
+*/
+static void memstatClearSchema(memstat_cursor *pCur){
+  int i;
+  if( pCur->azDb==0 ) return;
+  for(i=0; i<pCur->nDb; i++){
+    sqlite3_free(pCur->azDb[i]);
+  }
+  sqlite3_free(pCur->azDb);
+  pCur->azDb = 0;
+  pCur->nDb = 0;
+}
+
+/*
+** Fill in the azDb[] array for the cursor.
+*/
+static int memstatFindSchemas(memstat_cursor *pCur){
+  sqlite3_stmt *pStmt = 0;
+  int rc;
+  if( pCur->nDb ) return SQLITE_OK;
+  rc = sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pStmt, 0);
+  if( rc ){
+    sqlite3_finalize(pStmt);
+    return rc;
+  }
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    char **az, *z;
+    az = sqlite3_realloc(pCur->azDb, sizeof(char*)*(pCur->nDb+1));
+    if( az==0 ){
+      memstatClearSchema(pCur);
+      return SQLITE_NOMEM;
+    }
+    pCur->azDb = az;
+    z = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
+    if( z==0 ){
+      memstatClearSchema(pCur);
+      return SQLITE_NOMEM;
+    }
+    pCur->azDb[pCur->nDb] = z;
+    pCur->nDb++;
+  }
+  sqlite3_finalize(pStmt);
+  return SQLITE_OK;
+}
+
+
+/*
+** Destructor for a memstat_cursor.
+*/
+static int memstatClose(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  memstatClearSchema(pCur);
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Allowed values for aMemstatColumn[].eType
+*/
+#define MSV_GSTAT   0          /* sqlite3_status64() information */
+#define MSV_DB      1          /* sqlite3_db_status() information */
+#define MSV_ZIPVFS  2          /* ZIPVFS file-control with 64-bit return */
+
+/*
+** An array of quantities that can be measured and reported by
+** this virtual table
+*/
+static const struct MemstatColumns {
+  const char *zName;    /* Symbolic name */
+  unsigned char eType;  /* Type of interface */
+  unsigned char mNull;  /* Bitmask of which columns are NULL */
+                        /* 2: dbname,  4: current,  8: hiwtr */
+  int eOp;              /* Opcode */
+} aMemstatColumn[] = {
+ {"MEMORY_USED",            MSV_GSTAT,  2, SQLITE_STATUS_MEMORY_USED          },
+ {"MALLOC_SIZE",            MSV_GSTAT,  6, SQLITE_STATUS_MALLOC_SIZE          },
+ {"MALLOC_COUNT",           MSV_GSTAT,  2, SQLITE_STATUS_MALLOC_COUNT         },
+ {"PAGECACHE_USED",         MSV_GSTAT,  2, SQLITE_STATUS_PAGECACHE_USED       },
+ {"PAGECACHE_OVERFLOW",     MSV_GSTAT,  2, SQLITE_STATUS_PAGECACHE_OVERFLOW   },
+ {"PAGECACHE_SIZE",         MSV_GSTAT,  6, SQLITE_STATUS_PAGECACHE_SIZE       },
+ {"PARSER_STACK",           MSV_GSTAT,  6, SQLITE_STATUS_PARSER_STACK         },
+ {"DB_LOOKASIDE_USED",      MSV_DB,     2, SQLITE_DBSTATUS_LOOKASIDE_USED     },
+ {"DB_LOOKASIDE_HIT",       MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_HIT      },
+ {"DB_LOOKASIDE_MISS_SIZE", MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE},
+ {"DB_LOOKASIDE_MISS_FULL", MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL},
+ {"DB_CACHE_USED",          MSV_DB,    10, SQLITE_DBSTATUS_CACHE_USED         },
+#if SQLITE_VERSION_NUMBER >= 3140000
+ {"DB_CACHE_USED_SHARED",   MSV_DB,    10, SQLITE_DBSTATUS_CACHE_USED_SHARED  },
+#endif
+ {"DB_SCHEMA_USED",         MSV_DB,    10, SQLITE_DBSTATUS_SCHEMA_USED        },
+ {"DB_STMT_USED",           MSV_DB,    10, SQLITE_DBSTATUS_STMT_USED          },
+ {"DB_CACHE_HIT",           MSV_DB,    10, SQLITE_DBSTATUS_CACHE_HIT          },
+ {"DB_CACHE_MISS",          MSV_DB,    10, SQLITE_DBSTATUS_CACHE_MISS         },
+ {"DB_CACHE_WRITE",         MSV_DB,    10, SQLITE_DBSTATUS_CACHE_WRITE        },
+#if SQLITE_VERSION_NUMBER >= 3230000
+ {"DB_CACHE_SPILL",         MSV_DB,    10, SQLITE_DBSTATUS_CACHE_SPILL        },
+#endif
+ {"DB_DEFERRED_FKS",        MSV_DB,    10, SQLITE_DBSTATUS_DEFERRED_FKS       },
+#ifdef SQLITE_ENABLE_ZIPVFS
+ {"ZIPVFS_CACHE_USED",      MSV_ZIPVFS, 8, 231454 },
+ {"ZIPVFS_CACHE_HIT",       MSV_ZIPVFS, 8, 231455 },
+ {"ZIPVFS_CACHE_MISS",      MSV_ZIPVFS, 8, 231456 },
+ {"ZIPVFS_CACHE_WRITE",     MSV_ZIPVFS, 8, 231457 },
+ {"ZIPVFS_DIRECT_READ",     MSV_ZIPVFS, 8, 231458 },
+ {"ZIPVFS_DIRECT_BYTES",    MSV_ZIPVFS, 8, 231459 },
+#endif /* SQLITE_ENABLE_ZIPVFS */
+};
+#define MSV_NROW (sizeof(aMemstatColumn)/sizeof(aMemstatColumn[0]))
+
+/*
+** Advance a memstat_cursor to its next row of output.
+*/
+static int memstatNext(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  int i;
+  assert( pCur->iRowid<=MSV_NROW );
+  while(1){
+    i = (int)pCur->iRowid - 1;
+    if( i<0 || (aMemstatColumn[i].mNull & 2)!=0 || (++pCur->iDb)>=pCur->nDb ){
+      pCur->iRowid++;
+      if( pCur->iRowid>MSV_NROW ) return SQLITE_OK;  /* End of the table */
+      pCur->iDb = 0;
+      i++;
+    }
+    pCur->aVal[0] = 0;
+    pCur->aVal[1] = 0;    
+    switch( aMemstatColumn[i].eType ){
+      case MSV_GSTAT: {
+        if( sqlite3_libversion_number()>=3010000 ){
+          sqlite3_status64(aMemstatColumn[i].eOp,
+                           &pCur->aVal[0], &pCur->aVal[1],0);
+        }else{
+          int xCur, xHiwtr;
+          sqlite3_status(aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0);
+          pCur->aVal[0] = xCur;
+          pCur->aVal[1] = xHiwtr;
+        }
+        break;
+      }
+      case MSV_DB: {
+        int xCur, xHiwtr;
+        sqlite3_db_status(pCur->db, aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0);
+        pCur->aVal[0] = xCur;
+        pCur->aVal[1] = xHiwtr;
+        break;
+      }
+      case MSV_ZIPVFS: {
+        int rc;
+        rc = sqlite3_file_control(pCur->db, pCur->azDb[pCur->iDb],
+                                  aMemstatColumn[i].eOp, (void*)&pCur->aVal[0]);
+        if( rc!=SQLITE_OK ) continue;
+        break;
+      }
+    }
+    break;
+  }
+  return SQLITE_OK;
+}
+  
+
+/*
+** Return values of columns for the row at which the memstat_cursor
+** is currently pointing.
+*/
+static int memstatColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int iCol                    /* Which column to return */
+){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  int i;
+  assert( pCur->iRowid>0 && pCur->iRowid<=MSV_NROW );
+  i = (int)pCur->iRowid - 1;
+  if( (aMemstatColumn[i].mNull & (1<<iCol))!=0 ){
+    return SQLITE_OK;
+  }
+  switch( iCol ){
+    case MSV_COLUMN_NAME: {
+      sqlite3_result_text(ctx, aMemstatColumn[i].zName, -1, SQLITE_STATIC);
+      break;
+    }
+    case MSV_COLUMN_SCHEMA: {
+      sqlite3_result_text(ctx, pCur->azDb[pCur->iDb], -1, 0);
+      break;
+    }
+    case MSV_COLUMN_VALUE: {
+      sqlite3_result_int64(ctx, pCur->aVal[0]);
+      break;
+    }
+    case MSV_COLUMN_HIWTR: {
+      sqlite3_result_int64(ctx, pCur->aVal[1]);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int memstatRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  *pRowid = pCur->iRowid*1000 + pCur->iDb;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int memstatEof(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  return pCur->iRowid>MSV_NROW;
+}
+
+/*
+** This method is called to "rewind" the memstat_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to memstatColumn() or memstatRowid() or 
+** memstatEof().
+*/
+static int memstatFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  memstat_cursor *pCur = (memstat_cursor *)pVtabCursor;
+  int rc = memstatFindSchemas(pCur);
+  if( rc ) return rc;
+  pCur->iRowid = 0;
+  pCur->iDb = 0;
+  return memstatNext(pVtabCursor);
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the memstat virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int memstatBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = (double)500;
+  pIdxInfo->estimatedRows = 500;
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** memstat virtual table.
+*/
+static sqlite3_module memstatModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  memstatConnect,            /* xConnect */
+  memstatBestIndex,          /* xBestIndex */
+  memstatDisconnect,         /* xDisconnect */
+  0,                         /* xDestroy */
+  memstatOpen,               /* xOpen - open a cursor */
+  memstatClose,              /* xClose - close a cursor */
+  memstatFilter,             /* xFilter - configure scan constraints */
+  memstatNext,               /* xNext - advance a cursor */
+  memstatEof,                /* xEof - check for end of scan */
+  memstatColumn,             /* xColumn - read data */
+  memstatRowid,              /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+int sqlite3MemstatVtabInit(sqlite3 *db){
+  int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3_create_module(db, "sqlite_memstat", &memstatModule, 0);
+#endif
+  return rc;
+}
+
+#ifndef SQLITE_CORE
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_memstat_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3MemstatVtabInit(db);
+#endif
+  return rc;
+}
+#endif /* SQLITE_CORE */
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB) */
@@ -313,44 +313,45 @@ static int seriesBestIndex(
  sqlite3_vtab *tab,
  sqlite3_index_info *pIdxInfo
 ){
-  int i;                 /* Loop over constraints */
+  int i, j;              /* Loop over constraints */
  int idxNum = 0;        /* The query plan bitmask */
-  int startIdx = -1;     /* Index of the start= constraint, or -1 if none */
-  int stopIdx = -1;      /* Index of the stop= constraint, or -1 if none */
-  int stepIdx = -1;      /* Index of the step= constraint, or -1 if none */
+  int unusableMask = 0;  /* Mask of unusable constraints */
  int nArg = 0;          /* Number of arguments that seriesFilter() expects */
-
+  int aIdx[3];           /* Constraints on start, stop, and step */
  const struct sqlite3_index_constraint *pConstraint;
+
+  /* This implementation assumes that the start, stop, and step columns
+  ** are the last three columns in the virtual table. */
+  assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 );
+  assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 );
+  aIdx[0] = aIdx[1] = aIdx[2] = -1;
  pConstraint = pIdxInfo->aConstraint;
  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->usable==0 ) continue;
-    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    switch( pConstraint->iColumn ){
-      case SERIES_COLUMN_START:
-        startIdx = i;
-        idxNum |= 1;
-        break;
-      case SERIES_COLUMN_STOP:
-        stopIdx = i;
-        idxNum |= 2;
-        break;
-      case SERIES_COLUMN_STEP:
-        stepIdx = i;
-        idxNum |= 4;
-        break;
+    int iCol;    /* 0 for start, 1 for stop, 2 for step */
+    int iMask;   /* bitmask for those column */
+    if( pConstraint->iColumn<SERIES_COLUMN_START ) continue;
+    iCol = pConstraint->iColumn - SERIES_COLUMN_START;
+    assert( iCol>=0 && iCol<=2 );
+    iMask = 1 << iCol;
+    if( pConstraint->usable==0 ){
+      unusableMask |=  iMask;
+      continue;
+    }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      idxNum |= iMask;
+      aIdx[iCol] = i;
    }
  }
-  if( startIdx>=0 ){
-    pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY;
+  for(i=0; i<3; i++){
+    if( (j = aIdx[i])>=0 ){
+      pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg;
+      pIdxInfo->aConstraintUsage[j].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+    }
  }
-  if( stopIdx>=0 ){
-    pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
-  }
-  if( stepIdx>=0 ){
-    pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+  if( (unusableMask & ~idxNum)!=0 ){
+    /* The start, stop, and step columns are inputs.  Therefore if there
+    ** are unusable constraints on any of start, stop, or step then
+    ** this plan is unusable */
+    return SQLITE_CONSTRAINT;
  }
  if( (idxNum & 3)==3 ){
    /* Both start= and stop= boundaries are available.  This is the 
@@ -365,7 +366,6 @@ static int seriesBestIndex(
    /* If either boundary is missing, we have to generate a huge span
    ** of numbers.  Make this case very expensive so that the query
    ** planner will work hard to avoid it. */
-    pIdxInfo->estimatedCost = (double)2147483647;
    pIdxInfo->estimatedRows = 2147483647;
  }
  pIdxInfo->idxNum = idxNum;
@@ -10,7 +10,7 @@
 **
 ******************************************************************************
 **
-** This SQLite extension implements a functions that compute SHA1 hashes.
+** This SQLite extension implements functions that compute SHA1 hashes.
 ** Two SQL functions are implemented:
 **
 **     sha1(X)
@@ -10,7 +10,7 @@
 **
 ******************************************************************************
 **
-** This SQLite extension implements a functions that compute SHA1 hashes.
+** This SQLite extension implements functions that compute SHA3 hashes.
 ** Two SQL functions are implemented:
 **
 **     sha3(X,SIZE)
@@ -1295,6 +1295,9 @@ typedef struct Transliteration Transliteration;
 struct Transliteration {
 unsigned short int cFrom;
 unsigned char cTo0, cTo1, cTo2, cTo3;
+#ifdef SQLITE_SPELLFIX_5BYTE_MAPPINGS
+ unsigned char cTo4;
+#endif
 };

 /*
@@ -1708,7 +1711,11 @@ static const Transliteration *spellfixFindTranslit(int c, int *pxTop){
 ** should be freed by the caller.
 */
 static unsigned char *transliterate(const unsigned char *zIn, int nIn){
+#ifdef SQLITE_SPELLFIX_5BYTE_MAPPINGS
+  unsigned char *zOut = sqlite3_malloc64( nIn*5 + 1 );
+#else
  unsigned char *zOut = sqlite3_malloc64( nIn*4 + 1 );
+#endif
  int c, sz, nOut;
  if( zOut==0 ) return 0;
  nOut = 0;
@@ -1732,6 +1739,11 @@ static unsigned char *transliterate(const unsigned char *zIn, int nIn){
              zOut[nOut++] = tbl[x].cTo2;
              if( tbl[x].cTo3 ){
                zOut[nOut++] = tbl[x].cTo3;
+#ifdef SQLITE_SPELLFIX_5BYTE_MAPPINGS
+                if( tbl[x].cTo4 ){
+                  zOut[nOut++] = tbl[x].cTo4;
+                }
+#endif /* SQLITE_SPELLFIX_5BYTE_MAPPINGS */
              }
            }
          }
@@ -266,6 +266,7 @@ static sqlite3_module stmtModule = {
  0,                         /* xSavepoint */
  0,                         /* xRelease */
  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
 };

 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -248,7 +248,8 @@ static sqlite3_module templatevtabModule = {
  /* xRename     */ 0,
  /* xSavepoint  */ 0,
  /* xRelease    */ 0,
-  /* xRollbackTo */ 0
+  /* xRollbackTo */ 0,
+  /* xShadowName */ 0
 };


@@ -1350,7 +1350,8 @@ static int createUnionVtab(sqlite3 *db){
    0,                            /* xRename */
    0,                            /* xSavepoint */
    0,                            /* xRelease */
-    0                             /* xRollbackTo */
+    0,                            /* xRollbackTo */
+    0                             /* xShadowName */
  };
  int rc;

@@ -492,6 +492,7 @@ static sqlite3_module vtablogModule = {
  0,                         /* xSavepoint */
  0,                         /* xRelease */
  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
 };

 #ifdef _WIN32
@@ -1296,25 +1296,26 @@ static int zipfileBestIndex(
  sqlite3_index_info *pIdxInfo
 ){
  int i;
+  int idx = -1;
+  int unusable = 0;

  for(i=0; i<pIdxInfo->nConstraint; i++){
    const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
-    if( pCons->usable==0 ) continue;
-    if( pCons->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
    if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue;
-    break;
+    if( pCons->usable==0 ){
+      unusable = 1;
+    }else if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      idx = i;
+    }
  }
-
-  if( i<pIdxInfo->nConstraint ){
-    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
-    pIdxInfo->aConstraintUsage[i].omit = 1;
+  if( idx>=0 ){
+    pIdxInfo->aConstraintUsage[idx].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[idx].omit = 1;
    pIdxInfo->estimatedCost = 1000.0;
    pIdxInfo->idxNum = 1;
-  }else{
-    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
-    pIdxInfo->idxNum = 0;
+  }else if( unusable ){
+    return SQLITE_CONSTRAINT;
  }
-
  return SQLITE_OK;
 }

@@ -670,4 +670,3 @@ foreach {tn3 create_vfs destroy_vfs} {


 finish_test
-
@@ -185,4 +185,3 @@ do_test 4.3 {


 finish_test
-
@@ -195,4 +195,3 @@ do_test 4.7.2 {
 } {1 {SQLITE_ERROR - rbu_state mismatch error}}

 finish_test
-
@@ -232,4 +232,3 @@ do_multiclient_test tn {
 }

 finish_test
-
@@ -62,4 +62,3 @@ do_execsql_test 1.4 {


 finish_test
-
@@ -92,4 +92,3 @@ foreach {tn schema} {


 finish_test
-
@@ -203,5 +203,3 @@ do_test 5.3 {
 do_test 6.1 { sqlite3rbu_internal_test } {}

 finish_test
-
-
@@ -300,7 +300,3 @@ foreach {tn idx} {


 finish_test
-
-
-
-
@@ -100,4 +100,3 @@ for {set nStep 8} {$nStep < 20} {incr nStep} {


 finish_test
-
@@ -106,5 +106,3 @@ foreach {tn tbl} {
 }

 finish_test
-
-
@@ -72,4 +72,3 @@ integrity_check 1.3.3


 finish_test
-
@@ -125,4 +125,3 @@ foreach {tn idx} {


 finish_test
-
@@ -80,4 +80,3 @@ do_test 2.2 {


 finish_test
-
@@ -59,4 +59,3 @@ sqlite3_shutdown
 test_sqlite3_log 
 sqlite3_initialize
 finish_test
-
@@ -139,4 +139,3 @@ foreach {tn schema} {


 finish_test
-
@@ -86,12 +86,13 @@ proc step_rbu_legacy {target rbu} {
  set rc
 }

-proc do_rbu_vacuum_test {tn step} {
-  forcedelete state.db
-  uplevel [list do_test $tn.1 {
-    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
+proc do_rbu_vacuum_test {tn step {statedb state.db}} {
+  forcedelete $statedb
+  if {$statedb=="" && $step==1} breakpoint
+  uplevel [list do_test $tn.1 [string map [list %state% $statedb] {
+    if {$step==0} { sqlite3rbu_vacuum rbu test.db {%state%}}
    while 1 {
-      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
+      if {$step==1} { sqlite3rbu_vacuum rbu test.db {%state%}}
      set state [rbu state]
      check_prestep_state test.db $state
      set rc [rbu step]
@@ -100,7 +101,7 @@ proc do_rbu_vacuum_test {tn step} {
      if {$step==1} { rbu close }
    }
    rbu close
-  } {SQLITE_DONE}]
+  }] {SQLITE_DONE}]

  uplevel [list do_execsql_test $tn.2 {
    PRAGMA integrity_check
@@ -60,4 +60,3 @@ do_test 1.2 {

 #forcedelete testrbu.db
 finish_test
-
@@ -145,4 +145,3 @@ for {set nPre 0} {$nPre < $rbu_num_steps} {incr nPre} {
 }

 finish_test
-
@@ -103,4 +103,3 @@ for {set x 1} {$x < 2} {incr x} {
 }

 finish_test
-
@@ -300,4 +300,3 @@ tablE t1 USING FTs5(c);


 finish_test
-
@@ -56,4 +56,3 @@ do_execsql_test 1.4 {
 } [list 1 $bigA 2 $bigB]

 finish_test
-
@@ -234,4 +234,3 @@ foreach {tn2 setup sql expect} {
 }

 finish_test
-
@@ -55,4 +55,3 @@ do_faultsim_test 1 -faults oom* -prep {


 finish_test
-
@@ -95,4 +95,3 @@ foreach {fault errlist} {
 }

 finish_test
-
@@ -63,4 +63,3 @@ for {set tn 1} {1} {incr tn} {


 finish_test
-
@@ -131,4 +131,3 @@ do_test 3.3 {


 finish_test
-
@@ -172,4 +172,3 @@ for {set i 0} {$i<=3} {incr i} {


 finish_test
-
@@ -416,4 +416,3 @@ foreach {bReopen} { 0 1 } {


 finish_test
-
@@ -251,4 +251,3 @@ for {set n 1} {$n < 5000} {incr n} {
 }

 finish_test
-
@@ -102,4 +102,3 @@ do_execsql_test 1.5 {
 } {1 one 1 3 3 3 4 4 4 1 one 1 3 3 3 4 4 4}

 finish_test
-
@@ -92,4 +92,3 @@ foreach {tn cmd} {
 }

 finish_test
-
@@ -126,4 +126,3 @@ do_test 1.6.1 {
 do_test 1.6.2 { info commands rbu } {}

 finish_test
-
@@ -397,4 +397,3 @@ do_test 3.5 {

 catch { db close }
 finish_test
-
@@ -16,8 +16,11 @@

 source [file join [file dirname [info script]] rbu_common.tcl]

-foreach step {0 1} {
-  set ::testprefix rbuvacuum2-$step
+foreach {step} {0 1} {
+foreach {ttt state} {
+  s state.db t test.db-vacuum n {}
+} {
+  set ::testprefix rbuvacuum2-$step$ttt
  
  #-------------------------------------------------------------------------
  # Test that a database that contains fts3 tables can be vacuumed.
@@ -29,7 +32,7 @@ foreach step {0 1} {
      INSERT INTO t1 VALUES('fix this issue', 'at some point');
    }
  
-    do_rbu_vacuum_test 1.2 $step
+    do_rbu_vacuum_test 1.2 $step $state
  
    do_execsql_test 1.3 {
      SELECT * FROM t1;
@@ -46,7 +49,7 @@ foreach step {0 1} {
      INSERT INTO t1 VALUES('a b c', 'x y z');
    }

-    do_rbu_vacuum_test 1.6 $step
+    do_rbu_vacuum_test 1.6 $step $state
    do_execsql_test 1.7 {
      INSERT INTO t1(t1) VALUES('integrity-check');
      SELECT * FROM t1;
@@ -67,7 +70,7 @@ foreach step {0 1} {
      INSERT INTO t1 VALUES('fix this issue', 'at some point');
    }
  
-    do_rbu_vacuum_test 2.2 $step
+    do_rbu_vacuum_test 2.2 $step $state
  
    do_execsql_test 2.3 {
      SELECT * FROM t1;
@@ -84,7 +87,7 @@ foreach step {0 1} {
      INSERT INTO t1 VALUES('a b c', 'x y z');
    }

-    do_rbu_vacuum_test 2.6 $step
+    do_rbu_vacuum_test 2.6 $step $state
    do_execsql_test 2.7 {
      INSERT INTO t1(t1) VALUES('integrity-check');
      SELECT * FROM t1;
@@ -107,7 +110,7 @@ foreach step {0 1} {
      INSERT INTO rt VALUES(3, 55, 65);
    }
  
-    do_rbu_vacuum_test 3.2 $step
+    do_rbu_vacuum_test 3.2 $step $state
  
    do_execsql_test 3.3 {
      SELECT * FROM rt;
@@ -120,7 +123,7 @@ foreach step {0 1} {
      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
    } {2 3}

-    do_rbu_vacuum_test 3.5 $step
+    do_rbu_vacuum_test 3.5 $step $state

    do_execsql_test 3.6.1 {
      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
@@ -147,7 +150,7 @@ foreach step {0 1} {
    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
    }

-    do_rbu_vacuum_test 4.3 $step
+    do_rbu_vacuum_test 4.3 $step $state
    do_execsql_test 4.4 {
      SELECT * FROM sqlite_master;
    } {
@@ -157,6 +160,7 @@ foreach step {0 1} {
    }
  }
 }
+}
  
 #-------------------------------------------------------------------------
 # Test that passing a NULL value as the second argument to 
@@ -231,5 +235,9 @@ do_test 6.3 {
  execsql { PRAGMA integrity_check }
 } {ok}

-finish_test
+do_test 6.4 {
+  sqlite3rbu_vacuum rbu test.db test.db-vactmp
+  list [catch { rbu close } msg] $msg
+} {1 SQLITE_MISUSE}

+finish_test
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .25.0
 .27.0
				`@@ -670,4 +670,3 @@ foreach {tn3 create_vfs destroy_vfs} {`


				`finish_test`
				`@@ -100,4 +100,3 @@ for {set nStep 8} {$nStep < 20} {incr nStep} {`


				`finish_test`
				`@@ -55,4 +55,3 @@ do_faultsim_test 1 -faults oom* -prep {`


				`finish_test`
				`@@ -63,4 +63,3 @@ for {set tn 1} {1} {incr tn} {`


				`finish_test`
				`@@ -172,4 +172,3 @@ for {set i 0} {$i<=3} {incr i} {`


				`finish_test`
				`@@ -416,4 +416,3 @@ foreach {bReopen} { 0 1 } {`


				`finish_test`