When compiled with SQLITE_ENABLE_NORMALIZE, always have sqlite3_exec() enable the SQLITE_PREPARE_NORMALIZE flag.

FossilOrigin-Name: 28643d837d057879b3067a18bb4c628d10ffc8c69b8f1870154566cb0b09cba5
Add the "index_usage" utility program.
2018-12-04 21:16:34 +00:00 · 2018-12-04 16:51:42 +00:00 · 2018-12-04 14:33:02 +00:00 · 2018-12-04 13:51:26 +00:00 · 2018-12-03 23:57:27 +00:00 · 2018-12-03 17:40:46 +00:00
489 changed files with 48247 additions and 6973 deletions
@@ -22,7 +22,7 @@ TOP = @abs_srcdir@
 #
 BCC = @BUILD_CC@ @BUILD_CFLAGS@

-# TCC is the C Compile and options for use in building executables that 
+# TCC is the C Compile and options for use in building executables that
 # will run on the target platform.  (BCC and TCC are usually the
 # same unless your are cross-compiling.)  Separate CC and CFLAGS macros
 # are provide so that these aspects of the build process can be changed
@@ -32,10 +32,11 @@ CC = @CC@
 CFLAGS = @CPPFLAGS@ @CFLAGS@
 TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
 TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
+TCC += -I${TOP}/ext/userauth

 # Define this for the autoconf-based build, so that the code knows it can
 # include the generated config.h
-# 
+#
 TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite

 # Define -DNDEBUG to compile without debugging (i.e., for production usage)
@@ -66,7 +67,7 @@ LIBREADLINE = @TARGET_READLINE_LIBS@
 TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@

 # Any target libraries which libsqlite must be linked against
-# 
+#
 TLIBS = @LIBS@ $(LIBS)

 # Flags controlling use of the in memory btree implementation
@@ -78,8 +79,8 @@ TLIBS = @LIBS@ $(LIBS)
 TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@

 # Enable/disable loadable extensions, and other optional features
-# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).  
-# The same set of OMIT and ENABLE flags should be passed to the 
+# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
+# The same set of OMIT and ENABLE flags should be passed to the
 # LEMON parser generator and the mkkeywordhash tool as well.
 OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@

@@ -126,8 +127,8 @@ SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
 # If gcov support was enabled by the configure script, add the appropriate
 # flags here.  It's not always as easy as just having the user add the right
 # CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
-# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.  
-# Supposedly GCC does the right thing if you use --coverage, but in 
+# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.
+# Supposedly GCC does the right thing if you use --coverage, but in
 # practice it still fails.  See:
 #
 # http://www.mail-archive.com/debian-gcc@lists.debian.org/msg26197.html
@@ -187,10 +188,10 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
         random.lo resolve.lo rowset.lo rtree.lo \
         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
         table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
-         update.lo util.lo vacuum.lo \
+         update.lo userauth.lo upsert.lo util.lo vacuum.lo \
         vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
         vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
-         utf.lo vtab.lo
+         window.lo utf.lo vtab.lo

 # Object files for the amalgamation.
 #
@@ -284,6 +285,7 @@ SRC = \
  $(TOP)/src/trigger.c \
  $(TOP)/src/utf.c \
  $(TOP)/src/update.c \
+  $(TOP)/src/upsert.c \
  $(TOP)/src/util.c \
  $(TOP)/src/vacuum.c \
  $(TOP)/src/vdbe.c \
@@ -303,7 +305,8 @@ SRC = \
  $(TOP)/src/where.c \
  $(TOP)/src/wherecode.c \
  $(TOP)/src/whereexpr.c \
-  $(TOP)/src/whereInt.h
+  $(TOP)/src/whereInt.h \
+  $(TOP)/src/window.c

 # Source code for extensions
 #
@@ -348,10 +351,14 @@ SRC += \
  $(TOP)/ext/icu/icu.c
 SRC += \
  $(TOP)/ext/rtree/rtree.h \
-  $(TOP)/ext/rtree/rtree.c
+  $(TOP)/ext/rtree/rtree.c \
+  $(TOP)/ext/rtree/geopoly.c
 SRC += \
  $(TOP)/ext/session/sqlite3session.c \
  $(TOP)/ext/session/sqlite3session.h
+SRC += \
+  $(TOP)/ext/userauth/userauth.c \
+  $(TOP)/ext/userauth/sqlite3userauth.h
 SRC += \
  $(TOP)/ext/rbu/sqlite3rbu.h \
  $(TOP)/ext/rbu/sqlite3rbu.c
@@ -417,11 +424,12 @@ TESTSRC = \
  $(TOP)/src/test_thread.c \
  $(TOP)/src/test_vfs.c \
  $(TOP)/src/test_windirent.c \
+  $(TOP)/src/test_window.c \
  $(TOP)/src/test_wsd.c       \
  $(TOP)/ext/fts3/fts3_term.c \
  $(TOP)/ext/fts3/fts3_test.c  \
  $(TOP)/ext/session/test_session.c \
-  $(TOP)/ext/rbu/test_rbu.c 
+  $(TOP)/ext/rbu/test_rbu.c

 # Statically linked extensions
 #
@@ -433,6 +441,7 @@ TESTSRC += \
  $(TOP)/ext/misc/closure.c \
  $(TOP)/ext/misc/csv.c \
  $(TOP)/ext/misc/eval.c \
+  $(TOP)/ext/misc/explain.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
@@ -450,7 +459,8 @@ TESTSRC += \
  $(TOP)/ext/misc/totype.c \
  $(TOP)/ext/misc/unionvtab.c \
  $(TOP)/ext/misc/wholenumber.c \
-  $(TOP)/ext/misc/zipfile.c
+  $(TOP)/ext/misc/zipfile.c \
+  $(TOP)/ext/userauth/userauth.c

 # Source code to the library files needed by the test fixture
 #
@@ -466,6 +476,7 @@ TESTSRC2 = \
  $(TOP)/src/dbstat.c \
  $(TOP)/src/expr.c \
  $(TOP)/src/func.c \
+  $(TOP)/src/global.c \
  $(TOP)/src/insert.c \
  $(TOP)/src/wal.c \
  $(TOP)/src/main.c \
@@ -492,6 +503,7 @@ TESTSRC2 = \
  $(TOP)/src/where.c \
  $(TOP)/src/wherecode.c \
  $(TOP)/src/whereexpr.c \
+  $(TOP)/src/window.c \
  parse.c \
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3_aux.c \
@@ -501,7 +513,7 @@ TESTSRC2 = \
  $(TOP)/ext/fts3/fts3_write.c \
  $(TOP)/ext/async/sqlite3async.c \
  $(TOP)/ext/session/sqlite3session.c \
-  $(TOP)/ext/misc/stmt.c 
+  $(TOP)/ext/misc/stmt.c

 # Header files used by all library source files.
 #
@@ -548,11 +560,14 @@ EXTHDR += \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_tokenizer.h
 EXTHDR += \
-  $(TOP)/ext/rtree/rtree.h
+  $(TOP)/ext/rtree/rtree.h \
+  $(TOP)/ext/rtree/geopoly.c
 EXTHDR += \
  $(TOP)/ext/icu/sqliteicu.h
 EXTHDR += \
  $(TOP)/ext/rtree/sqlite3rtree.h
+EXTHDR += \
+  $(TOP)/ext/userauth/sqlite3userauth.h

 # executables needed for testing
 #
@@ -571,7 +586,9 @@ FUZZDATA = \
  $(TOP)/test/fuzzdata2.db \
  $(TOP)/test/fuzzdata3.db \
  $(TOP)/test/fuzzdata4.db \
-  $(TOP)/test/fuzzdata5.db
+  $(TOP)/test/fuzzdata5.db \
+  $(TOP)/test/fuzzdata6.db \
+  $(TOP)/test/fuzzdata7.db

 # Standard options to testfixture
 #
@@ -588,13 +605,14 @@ SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
+SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE
 SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
-DBFUZZ_OPT = 
+DBFUZZ_OPT =

 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
@@ -656,6 +674,22 @@ sessionfuzz$(TEXE):	$(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h
 dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)

+DBFUZZ2_OPTS = \
+  -DSQLITE_THREADSAFE=0 \
+  -DSQLITE_OMIT_LOAD_EXTENSION \
+  -DSQLITE_ENABLE_DESERIALIZE \
+  -DSQLITE_DEBUG \
+  -DSQLITE_ENABLE_DBSTAT_VTAB \
+  -DSQLITE_ENABLE_RTREE \
+  -DSQLITE_ENABLE_FTS4 \
+  -DSQLITE_EANBLE_FTS5
+
+dbfuzz2:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
+	clang-6.0 -I. -g -O0 -fsanitize=fuzzer,undefined,address -o dbfuzz2 \
+		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c
+	mkdir -p dbfuzz2-dir
+	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
+
 mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
 	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
 		$(TLIBS) -rpath "$(libdir)"
@@ -914,6 +948,9 @@ trigger.lo:	$(TOP)/src/trigger.c $(HDR)
 update.lo:	$(TOP)/src/update.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/update.c

+upsert.lo:	$(TOP)/src/upsert.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/upsert.c
+
 utf.lo:	$(TOP)/src/utf.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/utf.c

@@ -962,6 +999,9 @@ wherecode.lo:	$(TOP)/src/wherecode.c $(HDR)
 whereexpr.lo:	$(TOP)/src/whereexpr.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/whereexpr.c

+window.lo:	$(TOP)/src/window.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/window.c
+
 tclsqlite.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c

@@ -1085,7 +1125,10 @@ fts3_write.lo:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
 rtree.lo:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c

-sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
+sqlite3session.lo:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c
+
+userauth.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c

 json1.lo:	$(TOP)/ext/misc/json1.c
@@ -1113,7 +1156,7 @@ FTS5_SRC = \
   $(TOP)/ext/fts5/fts5_varint.c \
   $(TOP)/ext/fts5/fts5_vocab.c  \

-fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
+fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon
 	cp $(TOP)/ext/fts5/fts5parse.y .
 	rm -f fts5parse.h
 	./lemon$(BEXE) $(OPTS) fts5parse.y
@@ -1140,7 +1183,7 @@ sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 #
 TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
 TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
-TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
+TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
@@ -1156,6 +1199,10 @@ testfixture$(TEXE):	$(TESTFIXTURE_SRC)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

+coretestprogs:	$(TESTPROGS)
+
+testprogs:	coretestprogs srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE)
+
 # A very detailed test running most or all test cases
 fulltest:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/all.test $(TESTOPTS)
@@ -1215,7 +1262,7 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)

-sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in	
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c

 sqltclsh$(TEXE): sqltclsh.c
@@ -1259,12 +1306,21 @@ showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
 showshm$(TEXE):	$(TOP)/tool/showshm.c
 	$(LTLINK) -o $@ $(TOP)/tool/showshm.c

+index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)
+
 changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)

+changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)
+
 rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)

+atrc$(TEXX): $(TOP)/test/atrc.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/test/atrc.c sqlite3.lo $(TLIBS)
+
 LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c

@@ -1279,7 +1335,7 @@ KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
 kvtest$(TEXE):	$(TOP)/test/kvtest.c sqlite3.c
 	$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)

-rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
+rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo
 	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)

 loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
@@ -1307,7 +1363,7 @@ snapshot-tarball: sqlite3.c
 # The next two rules are used to support the "threadtest" target. Building
 # threadtest runs a few thread-safety tests that are implemented in C. This
 # target is invoked by the releasetest.tcl script.
-# 
+#
 THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
                  $(TOP)/test/tt3_checkpoint.c \
                  $(TOP)/test/tt3_index.c      \
@@ -1321,7 +1377,7 @@ threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC)
 threadtest: threadtest3$(TEXE)
 	./threadtest3$(TEXE)

-releasetest:	
+releasetest:
 	$(TCLSH_CMD) $(TOP)/test/releasetest.tcl

 # Standard install and cleanup targets
@@ -1329,7 +1385,7 @@ releasetest:
 lib_install:	libsqlite3.la
 	$(INSTALL) -d $(DESTDIR)$(libdir)
 	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
-	
+
 install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
 	$(INSTALL) -d $(DESTDIR)$(bindir)
 	$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
@@ -1347,7 +1403,7 @@ tcl_install:	lib_install libtclsqlite3.la pkgIndex.tcl
 	rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a
 	$(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)

-clean:	
+clean:
 	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
 	rm -f sqlite3.h opcodes.*
 	rm -rf .libs .deps
@@ -339,6 +339,13 @@ SQLITE_TCL_DEP =
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
@@ -620,6 +627,10 @@ SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS)
 !IFNDEF SHELL_CORE_SRC
 !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 SHELL_CORE_SRC =
+# <<mark>>
+!ELSEIF $(USE_AMALGAMATION)==0
+SHELL_CORE_SRC =
+# <</mark>>
 !ELSE
 SHELL_CORE_SRC = $(SQLITE3C)
 !ENDIF
@@ -630,6 +641,10 @@ SHELL_CORE_SRC = $(SQLITE3C)
 !IFNDEF SHELL_CORE_DEP
 !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 SHELL_CORE_DEP = $(SQLITE3DLL)
+# <<mark>>
+!ELSEIF $(USE_AMALGAMATION)==0
+SHELL_CORE_DEP = libsqlite3.lib
+# <</mark>>
 !ELSE
 SHELL_CORE_DEP =
 !ENDIF
@@ -649,6 +664,10 @@ TESTFIXTURE_DEP = zlib $(TESTFIXTURE_DEP)
 !IFNDEF SHELL_CORE_LIB
 !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 SHELL_CORE_LIB = $(SQLITE3LIB)
+# <<mark>>
+!ELSEIF $(USE_AMALGAMATION)==0
+SHELL_CORE_LIB = libsqlite3.lib
+# <</mark>>
 !ELSE
 SHELL_CORE_LIB =
 !ENDIF
@@ -1184,19 +1203,19 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
         fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
         fts5.lo \
         func.lo global.lo hash.lo \
-         icu.lo insert.lo legacy.lo loadext.lo \
+         icu.lo insert.lo json1.lo legacy.lo loadext.lo \
         main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
         memdb.lo memjournal.lo \
         mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
         notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
         pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
         random.lo resolve.lo rowset.lo rtree.lo \
-         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
+         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
         table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
-         update.lo util.lo vacuum.lo \
+         update.lo upsert.lo util.lo vacuum.lo \
         vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
         vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
-         utf.lo vtab.lo
+         window.lo utf.lo vtab.lo
 # <</mark>>

 # Object files for the amalgamation.
@@ -1292,6 +1311,7 @@ SRC01 = \
  $(TOP)\src\trigger.c \
  $(TOP)\src\utf.c \
  $(TOP)\src\update.c \
+  $(TOP)\src\upsert.c \
  $(TOP)\src\util.c \
  $(TOP)\src\vacuum.c \
  $(TOP)\src\vdbe.c \
@@ -1306,7 +1326,8 @@ SRC01 = \
  $(TOP)\src\walker.c \
  $(TOP)\src\where.c \
  $(TOP)\src\wherecode.c \
-  $(TOP)\src\whereexpr.c
+  $(TOP)\src\whereexpr.c \
+  $(TOP)\src\window.c

 # Core miscellaneous files.
 #
@@ -1399,6 +1420,7 @@ SRC09 = \
  $(TOP)\ext\fts3\fts3_tokenizer.h \
  $(TOP)\ext\icu\sqliteicu.h \
  $(TOP)\ext\rtree\rtree.h \
+  $(TOP)\ext\rtree\geopoly.c \
  $(TOP)\ext\rbu\sqlite3rbu.h \
  $(TOP)\ext\session\sqlite3session.h

@@ -1477,6 +1499,7 @@ TESTSRC = \
  $(TOP)\src\test_thread.c \
  $(TOP)\src\test_vfs.c \
  $(TOP)\src\test_windirent.c \
+  $(TOP)\src\test_window.c \
  $(TOP)\src\test_wsd.c \
  $(TOP)\ext\fts3\fts3_term.c \
  $(TOP)\ext\fts3\fts3_test.c \
@@ -1493,6 +1516,7 @@ TESTEXT = \
  $(TOP)\ext\misc\closure.c \
  $(TOP)\ext\misc\csv.c \
  $(TOP)\ext\misc\eval.c \
+  $(TOP)\ext\misc\explain.c \
  $(TOP)\ext\misc\fileio.c \
  $(TOP)\ext\misc\fuzzer.c \
  $(TOP)\ext\fts5\fts5_tcl.c \
@@ -1572,7 +1596,8 @@ EXTHDR = $(EXTHDR) \
  $(TOP)\ext\fts3\fts3_hash.h \
  $(TOP)\ext\fts3\fts3_tokenizer.h
 EXTHDR = $(EXTHDR) \
-  $(TOP)\ext\rtree\rtree.h
+  $(TOP)\ext\rtree\rtree.h \
+  $(TOP)\ext\rtree\geopoly.c
 EXTHDR = $(EXTHDR) \
  $(TOP)\ext\icu\sqliteicu.h
 EXTHDR = $(EXTHDR) \
@@ -1598,17 +1623,19 @@ FUZZDATA = \
  $(TOP)\test\fuzzdata2.db \
  $(TOP)\test\fuzzdata3.db \
  $(TOP)\test\fuzzdata4.db \
-  $(TOP)\test\fuzzdata5.db
+  $(TOP)\test\fuzzdata5.db \
+  $(TOP)\test\fuzzdata6.db \
+  $(TOP)\test\fuzzdata7.db
 # <</mark>>

 # Additional compiler options for the shell.  These are only effective
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_INTROSPECTION_PRAGMAS
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_RTREE
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF

 # <<mark>>
@@ -1672,7 +1699,7 @@ $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
 sqlite3.def:	libsqlite3.lib
 	echo EXPORTS > sqlite3.def
 	dumpbin /all libsqlite3.lib \
-		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup)?_[^@]*)(?:@\d+)?$$" \1 \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@]*)(?:@\d+)?$$" \1 \
 		| sort >> sqlite3.def
 # <</block2>>

@@ -1693,8 +1720,8 @@ scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
 srcck1.exe:	$(TOP)\tool\srcck1.c
 	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c

-sourcetest:	srcck1.exe sqlite3.c
-	srcck1.exe sqlite3.c
+sourcetest:	srcck1.exe $(SQLITE3C)
+	srcck1.exe $(SQLITE3C)

 fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1993,6 +2020,9 @@ trigger.lo:	$(TOP)\src\trigger.c $(HDR)
 update.lo:	$(TOP)\src\update.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\update.c

+upsert.lo:	$(TOP)\src\upsert.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\upsert.c
+
 utf.lo:	$(TOP)\src\utf.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\utf.c

@@ -2041,6 +2071,9 @@ wherecode.lo:	$(TOP)\src\wherecode.c $(HDR)
 whereexpr.lo:	$(TOP)\src\whereexpr.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\whereexpr.c

+window.lo:	$(TOP)\src\window.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\window.c
+
 tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
 	$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c

@@ -2091,7 +2124,7 @@ keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 # Source files that go into making shell.c
 SHELL_SRC = \
 	$(TOP)\src\shell.c.in \
-        $(TOP)\ext\misc\appendvfs.c \
+	$(TOP)\ext\misc\appendvfs.c \
 	$(TOP)\ext\misc\shathree.c \
 	$(TOP)\ext\misc\fileio.c \
 	$(TOP)\ext\misc\completion.c \
@@ -2174,6 +2207,12 @@ fts3_unicode2.lo:	$(TOP)\ext\fts3\fts3_unicode2.c $(HDR) $(EXTHDR)
 fts3_write.lo:	$(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c

+json1.lo:	$(TOP)\ext\misc\json1.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\json1.c
+
+stmt.lo:	$(TOP)\ext\misc\stmt.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\stmt.c
+
 rtree.lo:	$(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c

@@ -2256,9 +2295,9 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
-TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
-TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB
-TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)

 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
@@ -2299,6 +2338,10 @@ extensiontest:	testfixture.exe testloadext.dll
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)

+coretestprogs:	$(TESTPROGS)
+
+testprogs:	coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe
+
 fulltest:	$(TESTPROGS) fuzztest
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\all.test $(TESTOPTS)
@@ -2401,11 +2444,20 @@ showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
 showshm.exe:	$(TOP)\tool\showshm.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)

+index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		$(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
 		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

+changesetfuzz.exe:	$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
+		$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -2414,6 +2466,10 @@ rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

+atrc.exe:	$(TOP)\test\atrc.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		$(TOP)\test\atrc.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 LogEst.exe:	$(TOP)\tool\logest.c $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)

@@ -2445,7 +2501,6 @@ clean:
 	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
 	del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL
 # <<mark>>
-	del /Q sqlite3.c sqlite3.h 2>NUL
 	del /Q opcodes.c opcodes.h 2>NUL
 	del /Q lemon.* lempar.c parse.* 2>NUL
 	del /Q mksourceid.* mkkeywordhash.* keywordhash.h 2>NUL
@@ -2462,7 +2517,7 @@ clean:
 	del /Q changeset.exe 2>NUL
 	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
 	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
-	del /Q sqlite3.c sqlite3-*.c 2>NUL
+	del /Q sqlite3.c sqlite3-*.c sqlite3.h 2>NUL
 	del /Q sqlite3rc.h 2>NUL
 	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
@@ -1,12 +1,14 @@
 <h1 align="center">SQLite Source Repository</h1>

-This repository contains the complete source code for the SQLite database
-engine.  Some test scripts are also included.  However, many other test scripts
+This repository contains the complete source code for the 
+[SQLite database engine](https://sqlite.org/).  Some test scripts 
+are also included.  However, many other test scripts
 and most of the documentation are managed separately.

-If you are reading this on a Git mirror someplace, you are doing it wrong.
-The [official repository](https://www.sqlite.org/src/) is better.  Go there
-now.
+SQLite [does not use Git](https://sqlite.org/whynotgit.html).
+If you are reading this on GitHub, then you are looking at an
+unofficial mirror. See <https://sqlite.org/src> for the official
+repository.

 ## Obtaining The Code

@@ -1 +1 @@
-3.23.0
+3.26.0
@@ -1,6 +1,5 @@

-AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @ZLIB_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
-
+AM_CFLAGS = @BUILD_CFLAGS@ 
 lib_LTLIBRARIES = libsqlite3.la
 libsqlite3_la_SOURCES = sqlite3.c
 libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
@@ -14,7 +13,7 @@ sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_D

 include_HEADERS = sqlite3.h sqlite3ext.h

-EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
+EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs Makefile.fallback
 pkgconfigdir = ${libdir}/pkgconfig
 pkgconfig_DATA = sqlite3.pc

@@ -0,0 +1,19 @@
+#!/usr/bin/make
+#
+# If the configure script does not work, then this Makefile is available
+# as a backup.  Manually configure the variables below.
+#
+# Note:  This makefile works out-of-the-box on MacOS 10.2 (Jaguar)
+#
+CC = gcc
+CFLAGS = -O0 -I.
+LIBS = -lz
+COPTS += -D_BSD_SOURCE
+COPTS += -DSQLITE_ENABLE_LOCKING_STYLE=0
+COPTS += -DSQLITE_THREADSAFE=0
+COPTS += -DSQLITE_OMIT_LOAD_EXTENSION
+COPTS += -DSQLITE_WITHOUT_ZONEMALLOC
+COPTS += -DSQLITE_ENABLE_RTREE
+
+sqlite3:	shell.c sqlite3.c
+	$(CC) $(CFLAGS) $(COPTS) -o sqlite3 shell.c sqlite3.c $(LIBS)
@@ -277,6 +277,13 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
@@ -928,10 +935,10 @@ LIBRESOBJS =
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_INTROSPECTION_PRAGMAS
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_RTREE
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF


@@ -966,7 +973,7 @@ Replace.exe:
 sqlite3.def:	Replace.exe $(LIBOBJ)
 	echo EXPORTS > sqlite3.def
 	dumpbin /all $(LIBOBJ) \
-		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
+		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
 		| sort >> sqlite3.def

 $(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
@@ -29,6 +29,7 @@ AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
 AC_FUNC_STRERROR_R

 AC_CONFIG_FILES([Makefile sqlite3.pc])
+BUILD_CFLAGS=
 AC_SUBST(BUILD_CFLAGS)

 #-------------------------------------------------------------------------
@@ -86,13 +87,11 @@ AC_SUBST(READLINE_LIBS)
 AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING(
  [--enable-threadsafe], [build a thread-safe library [default=yes]])], 
  [], [enable_threadsafe=yes])
-THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
 if test x"$enable_threadsafe" != "xno"; then
-  THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
+  BUILD_CFLAGS="$BUILD_CFLAGS -D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
  AC_SEARCH_LIBS(pthread_create, pthread)
  AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
 fi
-AC_SUBST(THREADSAFE_FLAGS)
 #-----------------------------------------------------------------------

 #-----------------------------------------------------------------------
@@ -104,36 +103,66 @@ AC_ARG_ENABLE(dynamic-extensions, [AS_HELP_STRING(
 if test x"$enable_dynamic_extensions" != "xno"; then
  AC_SEARCH_LIBS(dlopen, dl)
 else
-  DYNAMIC_EXTENSION_FLAGS=-DSQLITE_OMIT_LOAD_EXTENSION=1
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_OMIT_LOAD_EXTENSION=1"
 fi
 AC_MSG_CHECKING([for whether to support dynamic extensions])
 AC_MSG_RESULT($enable_dynamic_extensions)
-AC_SUBST(DYNAMIC_EXTENSION_FLAGS)
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-fts4
+#
+AC_ARG_ENABLE(fts4, [AS_HELP_STRING(
+  [--enable-fts4], [include fts4 support [default=yes]])], 
+  [], [enable_fts4=yes])
+if test x"$enable_fts4" = "xyes"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS4"
+fi
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-fts3
+#
+AC_ARG_ENABLE(fts3, [AS_HELP_STRING(
+  [--enable-fts3], [include fts3 support [default=no]])], 
+  [], [])
+if test x"$enable_fts3" = "xyes" -a x"$enable_fts4" = "xno"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS3"
+fi
 #-----------------------------------------------------------------------

 #-----------------------------------------------------------------------
 #   --enable-fts5
 #
 AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
-  [--enable-fts5], [include fts5 support [default=no]])], 
-  [], [enable_fts5=no])
+  [--enable-fts5], [include fts5 support [default=yes]])], 
+  [], [enable_fts5=yes])
 if test x"$enable_fts5" = "xyes"; then
  AC_SEARCH_LIBS(log, m)
-  FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS5"
 fi
-AC_SUBST(FTS5_FLAGS)
 #-----------------------------------------------------------------------

 #-----------------------------------------------------------------------
 #   --enable-json1
 #
 AC_ARG_ENABLE(json1, [AS_HELP_STRING(
-  [--enable-json1], [include json1 support [default=no]])], 
-  [], [enable_json1=no])
+  [--enable-json1], [include json1 support [default=yes]])], 
+  [],[enable_json1=yes])
 if test x"$enable_json1" = "xyes"; then
-  JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_JSON1"
+fi
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-rtree
+#
+AC_ARG_ENABLE(rtree, [AS_HELP_STRING(
+  [--enable-rtree], [include rtree support [default=yes]])], 
+  [], [enable_rtree=yes])
+if test x"$enable_rtree" = "xyes"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_RTREE"
 fi
-AC_SUBST(JSON1_FLAGS)
 #-----------------------------------------------------------------------

 #-----------------------------------------------------------------------
@@ -141,11 +170,22 @@ AC_SUBST(JSON1_FLAGS)
 #
 AC_ARG_ENABLE(session, [AS_HELP_STRING(
  [--enable-session], [enable the session extension [default=no]])], 
-  [], [enable_session=no])
+  [], [])
 if test x"$enable_session" = "xyes"; then
-  SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
+fi
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-debug
+#
+AC_ARG_ENABLE(debug, [AS_HELP_STRING(
+  [--enable-debug], [build with debugging features enabled [default=no]])], 
+  [], [])
+if test x"$enable_debug" = "xyes"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_DEBUG -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE"
+  CFLAGS="-g -O0"
 fi
-AC_SUBST(SESSION_FLAGS)
 #-----------------------------------------------------------------------

 #-----------------------------------------------------------------------
@@ -165,9 +205,8 @@ AC_SUBST(EXTRA_SHELL_OBJ)

 AC_CHECK_FUNCS(posix_fallocate)
 AC_CHECK_HEADERS(zlib.h,[
-  AC_SEARCH_LIBS(deflate,z,[ZLIB_FLAGS="-DSQLITE_HAVE_ZLIB"])
+  AC_SEARCH_LIBS(deflate,z,[BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_HAVE_ZLIB"])
 ])
-AC_SUBST(ZLIB_FLAGS)

 AC_SEARCH_LIBS(system,,,[SHELL_CFLAGS="-DSQLITE_NOHAVE_SYSTEM"])
 AC_SUBST(SHELL_CFLAGS)
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.23.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.26.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.23.0'
-PACKAGE_STRING='sqlite 3.23.0'
+PACKAGE_VERSION='3.26.0'
+PACKAGE_STRING='sqlite 3.26.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -911,6 +911,7 @@ enable_fts4
 enable_fts5
 enable_json1
 enable_update_limit
+enable_geopoly
 enable_rtree
 enable_session
 enable_gcov
@@ -1465,7 +1466,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.23.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.26.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1530,7 +1531,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.23.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.26.0:";;
   esac
  cat <<\_ACEOF

@@ -1563,6 +1564,7 @@ Optional Features:
  --enable-fts5           Enable the FTS5 extension
  --enable-json1          Enable the JSON1 extension
  --enable-update-limit   Enable the UPDATE/DELETE LIMIT clause
+  --enable-geopoly        Enable the GEOPOLY extension
  --enable-rtree          Enable the RTREE extension
  --enable-session        Enable the SESSION extension
  --enable-gcov           Enable coverage testing using gcov
@@ -1655,7 +1657,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.23.0
+sqlite configure 3.26.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2074,7 +2076,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.23.0, which was
+It was created by sqlite $as_me 3.26.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -3932,13 +3934,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
  lt_cv_nm_interface="BSD nm"
  echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3935: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3937: $ac_compile\"" >&5)
  (eval "$ac_compile" 2>conftest.err)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3938: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3940: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3941: output\"" >&5)
+  (eval echo "\"\$as_me:3943: output\"" >&5)
  cat conftest.out >&5
  if $GREP 'External.*some_variable' conftest.out > /dev/null; then
    lt_cv_nm_interface="MS dumpbin"
@@ -5144,7 +5146,7 @@ ia64-*-hpux*)
  ;;
 *-*-irix6*)
  # Find out which ABI we are using.
-  echo '#line 5147 "configure"' > conftest.$ac_ext
+  echo '#line 5149 "configure"' > conftest.$ac_ext
  if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  (eval $ac_compile) 2>&5
  ac_status=$?
@@ -6669,11 +6671,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6672: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6674: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:6676: \$? = $ac_status" >&5
+   echo "$as_me:6678: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7008,11 +7010,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7011: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7013: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:7015: \$? = $ac_status" >&5
+   echo "$as_me:7017: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7113,11 +7115,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7116: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7118: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7120: \$? = $ac_status" >&5
+   echo "$as_me:7122: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -7168,11 +7170,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7171: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7173: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7175: \$? = $ac_status" >&5
+   echo "$as_me:7177: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -9548,7 +9550,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9551 "configure"
+#line 9553 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -9644,7 +9646,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9647 "configure"
+#line 9649 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -10455,8 +10457,6 @@ fi
 # Check whether --enable-threadsafe was given.
 if test "${enable_threadsafe+set}" = set; then :
  enableval=$enable_threadsafe;
-else
-  enable_threadsafe=yes
 fi

 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support threadsafe operation" >&5
@@ -11249,12 +11249,10 @@ fi
 # check for debug enabled
 # Check whether --enable-debug was given.
 if test "${enable_debug+set}" = set; then :
-  enableval=$enable_debug; use_debug=$enableval
-else
-  use_debug=no
+  enableval=$enable_debug;
 fi

-if test "${use_debug}" = "yes" ; then
+if test "${enable_debug}" = "yes" ; then
  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 else
  TARGET_DEBUG="-DNDEBUG"
@@ -11265,12 +11263,10 @@ fi
 # See whether we should use the amalgamation to build
 # Check whether --enable-amalgamation was given.
 if test "${enable_amalgamation+set}" = set; then :
-  enableval=$enable_amalgamation; use_amalgamation=$enableval
-else
-  use_amalgamation=yes
+  enableval=$enable_amalgamation;
 fi

-if test "${use_amalgamation}" != "yes" ; then
+if test "${enable_amalgamation}" == "no" ; then
  USE_AMALGAMATION=0
 fi

@@ -11353,12 +11349,12 @@ fi
 # See whether we should allow loadable extensions
 # Check whether --enable-load-extension was given.
 if test "${enable_load_extension+set}" = set; then :
-  enableval=$enable_load_extension; use_loadextension=$enableval
+  enableval=$enable_load_extension;
 else
-  use_loadextension=yes
+  enable_load_extension=yes
 fi

-if test "${use_loadextension}" = "yes" ; then
+if test "${enable_load_extension}" = "yes" ; then
  OPT_FEATURE_FLAGS=""
  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing dlopen" >&5
 $as_echo_n "checking for library containing dlopen... " >&6; }
@@ -11425,9 +11421,7 @@ fi
 #
 # Check whether --enable-memsys5 was given.
 if test "${enable_memsys5+set}" = set; then :
-  enableval=$enable_memsys5; enable_memsys5=yes
-else
-  enable_memsys5=no
+  enableval=$enable_memsys5;
 fi

 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5
@@ -11442,9 +11436,7 @@ $as_echo "no" >&6; }
 fi
 # Check whether --enable-memsys3 was given.
 if test "${enable_memsys3+set}" = set; then :
-  enableval=$enable_memsys3; enable_memsys3=yes
-else
-  enable_memsys3=no
+  enableval=$enable_memsys3;
 fi

 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5
@@ -11462,9 +11454,7 @@ fi
 # See whether we should enable Full Text Search extensions
 # Check whether --enable-fts3 was given.
 if test "${enable_fts3+set}" = set; then :
-  enableval=$enable_fts3; enable_fts3=yes
-else
-  enable_fts3=no
+  enableval=$enable_fts3;
 fi

 if test "${enable_fts3}" = "yes" ; then
@@ -11472,9 +11462,7 @@ if test "${enable_fts3}" = "yes" ; then
 fi
 # Check whether --enable-fts4 was given.
 if test "${enable_fts4+set}" = set; then :
-  enableval=$enable_fts4; enable_fts4=yes
-else
-  enable_fts4=no
+  enableval=$enable_fts4;
 fi

 if test "${enable_fts4}" = "yes" ; then
@@ -11538,9 +11526,7 @@ fi
 fi
 # Check whether --enable-fts5 was given.
 if test "${enable_fts5+set}" = set; then :
-  enableval=$enable_fts5; enable_fts5=yes
-else
-  enable_fts5=no
+  enableval=$enable_fts5;
 fi

 if test "${enable_fts5}" = "yes" ; then
@@ -11607,9 +11593,7 @@ fi
 # See whether we should enable JSON1
 # Check whether --enable-json1 was given.
 if test "${enable_json1+set}" = set; then :
-  enableval=$enable_json1; enable_json1=yes
-else
-  enable_json1=no
+  enableval=$enable_json1;
 fi

 if test "${enable_json1}" = "yes" ; then
@@ -11621,22 +11605,32 @@ fi
 # statements.
 # Check whether --enable-update-limit was given.
 if test "${enable_update_limit+set}" = set; then :
-  enableval=$enable_update_limit; enable_udlimit=yes
-else
-  enable_udlimit=no
+  enableval=$enable_update_limit;
 fi

 if test "${enable_udlimit}" = "yes" ; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
 fi

+#########
+# See whether we should enable GEOPOLY
+# Check whether --enable-geopoly was given.
+if test "${enable_geopoly+set}" = set; then :
+  enableval=$enable_geopoly; enable_geopoly=yes
+else
+  enable_geopoly=no
+fi
+
+if test "${enable_geopoly}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY"
+  enable_rtree=yes
+fi
+
 #########
 # See whether we should enable RTREE
 # Check whether --enable-rtree was given.
 if test "${enable_rtree+set}" = set; then :
-  enableval=$enable_rtree; enable_rtree=yes
-else
-  enable_rtree=no
+  enableval=$enable_rtree;
 fi

 if test "${enable_rtree}" = "yes" ; then
@@ -11647,9 +11641,7 @@ fi
 # See whether we should enable the SESSION extension
 # Check whether --enable-session was given.
 if test "${enable_session+set}" = set; then :
-  enableval=$enable_session; enable_session=yes
-else
-  enable_session=no
+  enableval=$enable_session;
 fi

 if test "${enable_session}" = "yes" ; then
@@ -11712,9 +11704,7 @@ BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
 # See whether we should use GCOV
 # Check whether --enable-gcov was given.
 if test "${enable_gcov+set}" = set; then :
-  enableval=$enable_gcov; use_gcov=$enableval
-else
-  use_gcov=no
+  enableval=$enable_gcov;
 fi

 if test "${use_gcov}" = "yes" ; then
@@ -12242,7 +12232,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.23.0, which was
+This file was extended by sqlite $as_me 3.26.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12308,7 +12298,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.23.0
+sqlite config.status 3.26.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -182,7 +182,7 @@ AC_SUBST(BUILD_CC)
 # Do we want to support multithreaded use of sqlite
 #
 AC_ARG_ENABLE(threadsafe, 
-AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]),,enable_threadsafe=yes)
+AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]))
 AC_MSG_CHECKING([whether to support threadsafe operation])
 if test "$enable_threadsafe" = "no"; then
  SQLITE_THREADSAFE=0
@@ -557,9 +557,8 @@ AC_SEARCH_LIBS(fdatasync, [rt])

 #########
 # check for debug enabled
-AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
-      [use_debug=$enableval],[use_debug=no])
-if test "${use_debug}" = "yes" ; then
+AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
+if test "${enable_debug}" = "yes" ; then
  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 else
  TARGET_DEBUG="-DNDEBUG"
@@ -569,9 +568,8 @@ AC_SUBST(TARGET_DEBUG)
 #########
 # See whether we should use the amalgamation to build
 AC_ARG_ENABLE(amalgamation, AC_HELP_STRING([--disable-amalgamation],
-      [Disable the amalgamation and instead build all files separately]),
-      [use_amalgamation=$enableval],[use_amalgamation=yes])
-if test "${use_amalgamation}" != "yes" ; then
+      [Disable the amalgamation and instead build all files separately]))
+if test "${enable_amalgamation}" == "no" ; then
  USE_AMALGAMATION=0
 fi
 AC_SUBST(USE_AMALGAMATION)
@@ -585,9 +583,8 @@ AC_SUBST(HAVE_ZLIB)
 #########
 # See whether we should allow loadable extensions
 AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension],
-      [Disable loading of external extensions]),
-      [use_loadextension=$enableval],[use_loadextension=yes])
-if test "${use_loadextension}" = "yes" ; then
+      [Disable loading of external extensions]),,[enable_load_extension=yes])
+if test "${enable_load_extension}" = "yes" ; then
  OPT_FEATURE_FLAGS=""
  AC_SEARCH_LIBS(dlopen, dl)
 else
@@ -598,8 +595,7 @@ fi
 # Do we want to support memsys3 and/or memsys5
 #
 AC_ARG_ENABLE(memsys5, 
-  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]),
-  [enable_memsys5=yes],[enable_memsys5=no])
+  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]))
 AC_MSG_CHECKING([whether to support MEMSYS5])
 if test "${enable_memsys5}" = "yes"; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
@@ -608,8 +604,7 @@ else
  AC_MSG_RESULT([no])
 fi
 AC_ARG_ENABLE(memsys3, 
-  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]),
-  [enable_memsys3=yes],[enable_memsys3=no])
+  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]))
 AC_MSG_CHECKING([whether to support MEMSYS3])
 if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
@@ -621,21 +616,18 @@ fi
 #########
 # See whether we should enable Full Text Search extensions
 AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
-      [Enable the FTS3 extension]),
-      [enable_fts3=yes],[enable_fts3=no])
+      [Enable the FTS3 extension]))
 if test "${enable_fts3}" = "yes" ; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
 fi
 AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
-      [Enable the FTS4 extension]),
-      [enable_fts4=yes],[enable_fts4=no])
+      [Enable the FTS4 extension]))
 if test "${enable_fts4}" = "yes" ; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
  AC_SEARCH_LIBS([log],[m])
 fi
 AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
-      [Enable the FTS5 extension]),
-      [enable_fts5=yes],[enable_fts5=no])
+      [Enable the FTS5 extension]))
 if test "${enable_fts5}" = "yes" ; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
  AC_SEARCH_LIBS([log],[m])
@@ -643,9 +635,7 @@ fi

 #########
 # See whether we should enable JSON1
-AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],
-      [Enable the JSON1 extension]),
-      [enable_json1=yes],[enable_json1=no])
+AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],[Enable the JSON1 extension]))
 if test "${enable_json1}" = "yes" ; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
 fi
@@ -654,17 +644,25 @@ fi
 # See whether we should enable the LIMIT clause on UPDATE and DELETE
 # statements.
 AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
-      [Enable the UPDATE/DELETE LIMIT clause]),
-      [enable_udlimit=yes],[enable_udlimit=no])
+      [Enable the UPDATE/DELETE LIMIT clause]))
 if test "${enable_udlimit}" = "yes" ; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
 fi

+#########
+# See whether we should enable GEOPOLY
+AC_ARG_ENABLE(geopoly, AC_HELP_STRING([--enable-geopoly],
+      [Enable the GEOPOLY extension]),
+      [enable_geopoly=yes],[enable_geopoly=no])
+if test "${enable_geopoly}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY"
+  enable_rtree=yes
+fi
+
 #########
 # See whether we should enable RTREE
 AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
-      [Enable the RTREE extension]),
-      [enable_rtree=yes],[enable_rtree=no])
+      [Enable the RTREE extension]))
 if test "${enable_rtree}" = "yes" ; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
 fi
@@ -672,8 +670,7 @@ fi
 #########
 # See whether we should enable the SESSION extension
 AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
-      [Enable the SESSION extension]),
-      [enable_session=yes],[enable_session=no])
+      [Enable the SESSION extension]))
 if test "${enable_session}" = "yes" ; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
@@ -733,8 +730,7 @@ BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
 #########
 # See whether we should use GCOV
 AC_ARG_ENABLE(gcov, AC_HELP_STRING([--enable-gcov],
-      [Enable coverage testing using gcov]),
-      [use_gcov=$enableval],[use_gcov=no])
+      [Enable coverage testing using gcov]))
 if test "${use_gcov}" = "yes" ; then
  USE_GCOV=1
 else
@@ -0,0 +1,87 @@
+
+SQLite's OS layer contains the following definitions used in F2FS related
+calls:
+
+#define F2FS_IOCTL_MAGIC        0xf5
+#define F2FS_IOC_START_ATOMIC_WRITE     _IO(F2FS_IOCTL_MAGIC, 1)
+#define F2FS_IOC_COMMIT_ATOMIC_WRITE    _IO(F2FS_IOCTL_MAGIC, 2)
+#define F2FS_IOC_START_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 3)
+#define F2FS_IOC_ABORT_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 5)
+#define F2FS_IOC_GET_FEATURES           _IOR(F2FS_IOCTL_MAGIC, 12, u32)
+#define F2FS_FEATURE_ATOMIC_WRITE       0x0004
+
+After opening a database file on Linux (including Android), SQLite determines
+whether or not a file supports F2FS atomic commits as follows:
+
+  u32 flags = 0;
+  rc = ioctl(fd, F2FS_IOC_GET_FEATURES, &flags);
+  if( rc==0 && (flags & F2FS_FEATURE_ATOMIC_WRITE) ){
+    /* File supports F2FS atomic commits */
+  }else{
+    /* File does NOT support F2FS atomic commits */
+  }
+
+where "fd" is the file-descriptor open on the database file.
+
+Usually, when writing to a database file that supports atomic commits, SQLite
+accumulates the entire transaction in heap memory, deferring all writes to the
+db file until the transaction is committed.
+
+When it is time to commit a transaction on a file that supports atomic
+commits, SQLite does:
+
+  /* Take an F_WRLCK lock on the database file. This prevents any other
+  ** SQLite clients from reading or writing the file until the lock
+  ** is released.  */
+  rc = fcntl(fd, F_SETLK, ...);
+  if( rc!=0 ) goto failed;
+
+  rc = ioctl(fd, F2FS_IOC_START_ATOMIC_WRITE);
+  if( rc!=0 ) goto fallback_to_legacy_journal_commit;
+
+  foreach (dirty page){
+    rc = write(fd, ...dirty page...);
+    if( rc!=0 ){
+      ioctl(fd, F2FS_IOC_ABORT_VOLATILE_WRITE);
+      goto fallback_to_legacy_journal_commit;
+    }
+  }
+
+  rc = ioctl(fd, F2FS_IOC_COMMIT_ATOMIC_WRITE);
+  if( rc!=0 ){
+    ioctl(fd, F2FS_IOC_ABORT_VOLATILE_WRITE);
+    goto fallback_to_legacy_journal_commit;
+  }
+
+  /* If we get there, the transaction has been successfully 
+  ** committed to persistent storage. The following call
+  ** relinquishes the F_WRLCK lock.  */
+  fcntl(fd, F_SETLK, ...);
+
+Assumptions:
+
+1. After either of the F2FS_IOC_ABORT_VOLATILE_WRITE calls return,
+   the database file is in the state that it was in before
+   F2FS_IOC_START_ATOMIC_WRITE was invoked. Even if the ioctl()
+   fails - we're ignoring the return code.
+
+   This is true regardless of the type of error that occurred in
+   ioctl() or write().
+
+2. If the system fails before the F2FS_IOC_COMMIT_ATOMIC_WRITE is
+   completed, then following a reboot the database file is in the
+   state that it was in before F2FS_IOC_START_ATOMIC_WRITE was invoked.
+   Or, if the write was commited right before the system failed, in a 
+   state indicating that all write() calls were successfully committed
+   to persistent storage before the failure occurred.
+
+3. If the process crashes before the F2FS_IOC_COMMIT_ATOMIC_WRITE is
+   completed then the file is automatically restored to the state that
+   it was in before F2FS_IOC_START_ATOMIC_WRITE was called. This occurs
+   before the posix advisory lock is automatically dropped - there is
+   no chance that another client will be able to read the file in a
+   half-committed state before the rollback operation occurs.
+
+
+
+
@@ -99,6 +99,9 @@ Show only the basis for each parser state in the report file.
 <li><b>-c</b>
 Do not compress the generated action tables.  The parser will be a
 little larger and slower, but it will detect syntax errors sooner.
+<li><b>-d</b><i>directory</i>
+Write all output files into <i>directory</i>.  Normally, output files
+are written into the directory that contains the input grammar file.
 <li><b>-D<i>name</i></b>
 Define C preprocessor macro <i>name</i>.  This macro is usable by
 "<tt><a href='#pifdef'>%ifdef</a></tt>" and
@@ -679,6 +682,30 @@ of type "MyStruct*" and all action routines will have access to
 a variable named "pAbc" that is the value of the 4th parameter
 in the most recent call to Parse().</p>

+<p>The <tt>%extra_context</tt> directive works the same except that it
+is passed in on the ParseAlloc() or ParseInit() routines instead of
+on Parse().
+
+<a name='extractx'></a>
+<h4>The <tt>%extra_context</tt> directive</h4>
+
+The <tt>%extra_context</tt> directive instructs Lemon to add a 2th parameter
+to the parameter list of the ParseAlloc() and ParseInif() functions.  Lemon
+doesn't do anything itself with these extra argument, but it does
+store the value make it available to C-code action routines, destructors,
+and so forth.  For example, if the grammar file contains:</p>
+
+<p><pre>
+    %extra_context { MyStruct *pAbc }
+</pre></p>
+
+<p>Then the ParseAlloc() and ParseInit() functions will have an 2th parameter
+of type "MyStruct*" and all action routines will have access to
+a variable named "pAbc" that is the value of that 2th parameter.</p>
+
+<p>The <tt>%extra_argument</tt> directive works the same except that it
+is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().
+
 <a name='pfallback'></a>
 <h4>The <tt>%fallback</tt> directive</h4>

@@ -750,6 +777,9 @@ For example:</p>
 <p>This might be needed, for example, if some of the C actions in the
 grammar call functions that are prototyped in unistd.h.</p>

+<p>Use the <tt><a href="#pcode">%code</a></tt> directive to add code to
+the end of the generated parser.</p>
+
 <a name='pleft'></a>
 <h4>The <tt>%left</tt> directive</h4>

@@ -95,7 +95,7 @@ do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
  SELECT * FROM t1
 } {
  (no new indexes)
-  0|0|0|SCAN TABLE t1
+  SCAN TABLE t1
 }

 do_setup_rec_test $tn.2 {
@@ -104,7 +104,7 @@ do_setup_rec_test $tn.2 {
  SELECT * FROM t1 WHERE b>?;
 } {
  CREATE INDEX t1_idx_00000062 ON t1(b);
-  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
+  SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
 }

 do_setup_rec_test $tn.3 {
@@ -113,7 +113,7 @@ do_setup_rec_test $tn.3 {
  SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
 } {
  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
-  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
+  SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
 }

 do_setup_rec_test $tn.4 {
@@ -122,7 +122,7 @@ do_setup_rec_test $tn.4 {
  SELECT a FROM t1 ORDER BY b;
 } {
  CREATE INDEX t1_idx_00000062 ON t1(b);
-  0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
+  SCAN TABLE t1 USING INDEX t1_idx_00000062
 }

 do_setup_rec_test $tn.5 {
@@ -131,7 +131,7 @@ do_setup_rec_test $tn.5 {
  SELECT a FROM t1 WHERE a=? ORDER BY b;
 } {
  CREATE INDEX t1_idx_000123a7 ON t1(a, b);
-  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
+  SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
 }

 do_setup_rec_test $tn.6 {
@@ -140,7 +140,7 @@ do_setup_rec_test $tn.6 {
  SELECT min(a) FROM t1
 } {
  CREATE INDEX t1_idx_00000061 ON t1(a);
-  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
+  SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
 }

 do_setup_rec_test $tn.7 {
@@ -149,7 +149,7 @@ do_setup_rec_test $tn.7 {
  SELECT * FROM t1 ORDER BY a, b, c;
 } {
  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
-  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
+  SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
 }

 #do_setup_rec_test $tn.1.8 {
@@ -167,7 +167,7 @@ do_setup_rec_test $tn.8.1 {
  SELECT * FROM t1 WHERE a=?
 } {
  CREATE INDEX t1_idx_00000061 ON t1(a);
-  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
+  SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
 }
 do_setup_rec_test $tn.8.2 {
  CREATE TABLE t1(a, b COLLATE nocase, c);
@@ -175,7 +175,7 @@ do_setup_rec_test $tn.8.2 {
  SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
 } {
  CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
-  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
+  SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
 }


@@ -187,7 +187,7 @@ do_setup_rec_test $tn.9.1 {
  SELECT * FROM "t t" WHERE a=?
 } {
  CREATE INDEX 't t_idx_00000061' ON 't t'(a);
-  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
+  SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
 }

 do_setup_rec_test $tn.9.2 {
@@ -196,7 +196,7 @@ do_setup_rec_test $tn.9.2 {
  SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
 } {
  CREATE INDEX 't t_idx_00000062' ON 't t'(b);
-  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
+  SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
 }

 # Columns with names that require quotes.
@@ -207,7 +207,7 @@ do_setup_rec_test $tn.10.1 {
  SELECT * FROM t3 WHERE "b b" = ?
 } {
  CREATE INDEX t3_idx_00050c52 ON t3('b b');
-  0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
+  SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
 }

 do_setup_rec_test $tn.10.2 {
@@ -216,7 +216,7 @@ do_setup_rec_test $tn.10.2 {
  SELECT * FROM t3 ORDER BY "b b"
 } {
  CREATE INDEX t3_idx_00050c52 ON t3('b b');
-  0|0|0|SCAN TABLE t3 USING INDEX t3_idx_00050c52
+  SCAN TABLE t3 USING INDEX t3_idx_00050c52
 }

 # Transitive constraints
@@ -229,8 +229,8 @@ do_setup_rec_test $tn.11.1 {
 } {
  CREATE INDEX t5_idx_000123a7 ON t5(a, b);
  CREATE INDEX t6_idx_00000063 ON t6(c);
-  0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
-  0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
+  SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
+  SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
 }

 # OR terms.
@@ -242,8 +242,9 @@ do_setup_rec_test $tn.12.1 {
 } {
  CREATE INDEX t7_idx_00000062 ON t7(b);
  CREATE INDEX t7_idx_00000061 ON t7(a);
-  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
-  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
+  MULTI-INDEX OR
+    SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
+    SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }

 # rowid terms.
@@ -254,7 +255,7 @@ do_setup_rec_test $tn.13.1 {
  SELECT * FROM t8 WHERE rowid=?
 } {
  (no new indexes)
-  0|0|0|SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
+  SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
 }
 do_setup_rec_test $tn.13.2 {
  CREATE TABLE t8(a, b);
@@ -262,7 +263,7 @@ do_setup_rec_test $tn.13.2 {
  SELECT * FROM t8 ORDER BY rowid
 } {
  (no new indexes)
-  0|0|0|SCAN TABLE t8
+  SCAN TABLE t8
 }
 do_setup_rec_test $tn.13.3 {
  CREATE TABLE t8(a, b);
@@ -270,7 +271,7 @@ do_setup_rec_test $tn.13.3 {
  SELECT * FROM t8 WHERE a=? ORDER BY rowid
 } {
  CREATE INDEX t8_idx_00000061 ON t8(a); 
-  0|0|0|SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
+  SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
 }

 # Triggers
@@ -285,7 +286,7 @@ do_setup_rec_test $tn.14 {
  INSERT INTO t9 VALUES(?, ?, ?);
 } {
  CREATE INDEX t10_idx_00000062 ON t10(b); 
-  0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
+  SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
 }

 do_setup_rec_test $tn.15 {
@@ -301,8 +302,8 @@ do_setup_rec_test $tn.15 {
  SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
 } {
  CREATE INDEX t2_idx_00000064 ON t2(d);
-  0|0|0|SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
-  0|1|1|SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
+  SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
+  SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
 }

 do_setup_rec_test $tn.16 {
@@ -311,7 +312,7 @@ do_setup_rec_test $tn.16 {
  SELECT * FROM t1 WHERE b IS NOT NULL;
 } {
  (no new indexes)
-  0|0|0|SCAN TABLE t1
+  SCAN TABLE t1
 }

 }
@@ -331,7 +332,7 @@ proc do_candidates_test {tn sql res} {


 reset_db
-do_execsql_test 3.0 {
+do_execsql_test 4.0 {
  CREATE TABLE t1(a, b);
  CREATE TABLE t2(c, d);

@@ -341,7 +342,7 @@ do_execsql_test 3.0 {
  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
 }
-do_candidates_test 3.1 {
+do_candidates_test 4.1 {
  SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
 } {
  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
@@ -350,14 +351,14 @@ do_candidates_test 3.1 {
  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
 }

-do_candidates_test 3.2 {
+do_candidates_test 4.2 {
  SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=?
 } {
  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17
  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
 }

-do_execsql_test 3.2 {
+do_execsql_test 4.3 {
  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16
@@ -379,4 +380,3 @@ do_execsql_test 3.2 {


 finish_test
-
@@ -644,6 +644,7 @@ static int idxRegisterVtab(sqlite3expert *p){
    0,                            /* xSavepoint */
    0,                            /* xRelease */
    0,                            /* xRollbackTo */
+    0,                            /* xShadowName */
  };

  return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
@@ -1123,9 +1124,9 @@ int idxFindIndexes(
        "EXPLAIN QUERY PLAN %s", pStmt->zSql
    );
    while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
-      int iSelectid = sqlite3_column_int(pExplain, 0);
-      int iOrder = sqlite3_column_int(pExplain, 1);
-      int iFrom = sqlite3_column_int(pExplain, 2);
+      /* int iId = sqlite3_column_int(pExplain, 0); */
+      /* int iParent = sqlite3_column_int(pExplain, 1); */
+      /* int iNotUsed = sqlite3_column_int(pExplain, 2); */
      const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
      int nDetail = STRLEN(zDetail);
      int i;
@@ -1152,9 +1153,9 @@ int idxFindIndexes(
        }
      }

-      pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%d|%d|%d|%s\n", 
-          iSelectid, iOrder, iFrom, zDetail
-      );
+      if( zDetail[0]!='-' ){
+        pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%s\n", zDetail);
+      }
    }

    for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
@@ -1821,7 +1821,7 @@ static int fts3ScanInteriorNode(
  const char *zCsr = zNode;       /* Cursor to iterate through node */
  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
  char *zBuffer = 0;              /* Buffer to load terms into */
-  int nAlloc = 0;                 /* Size of allocated buffer */
+  i64 nAlloc = 0;                 /* Size of allocated buffer */
  int isFirstTerm = 1;            /* True when processing first term on page */
  sqlite3_int64 iChild;           /* Block id of child node to descend to */

@@ -1859,14 +1859,14 @@ static int fts3ScanInteriorNode(
    zCsr += fts3GetVarint32(zCsr, &nSuffix);
    
    assert( nPrefix>=0 && nSuffix>=0 );
-    if( &zCsr[nSuffix]>zEnd ){
+    if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
-    if( nPrefix+nSuffix>nAlloc ){
+    if( (i64)nPrefix+nSuffix>nAlloc ){
      char *zNew;
-      nAlloc = (nPrefix+nSuffix) * 2;
-      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+      nAlloc = ((i64)nPrefix+nSuffix) * 2;
+      zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
      if( !zNew ){
        rc = SQLITE_NOMEM;
        goto finish_scan;
@@ -3808,7 +3808,7 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  int rc = SQLITE_OK;
  UNUSED_PARAMETER(iSavepoint);
  assert( ((Fts3Table *)pVtab)->inTransaction );
-  assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
+  assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
  TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
  if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
    rc = fts3SyncMethod(pVtab);
@@ -3846,8 +3846,23 @@ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  return SQLITE_OK;
 }

+/*
+** Return true if zName is the extension on one of the shadow tables used
+** by this module.
+*/
+static int fts3ShadowName(const char *zName){
+  static const char *azName[] = {
+    "content", "docsize", "segdir", "segments", "stat", 
+  };
+  unsigned int i;
+  for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
+    if( sqlite3_stricmp(zName, azName[i])==0 ) return 1;
+  }
+  return 0;
+}
+
 static const sqlite3_module fts3Module = {
-  /* iVersion      */ 2,
+  /* iVersion      */ 3,
  /* xCreate       */ fts3CreateMethod,
  /* xConnect      */ fts3ConnectMethod,
  /* xBestIndex    */ fts3BestIndexMethod,
@@ -3870,6 +3885,7 @@ static const sqlite3_module fts3Module = {
  /* xSavepoint    */ fts3SavepointMethod,
  /* xRelease      */ fts3ReleaseMethod,
  /* xRollbackTo   */ fts3RollbackToMethod,
+  /* xShadowName   */ fts3ShadowName,
 };

 /*
@@ -3963,7 +3979,7 @@ int sqlite3Fts3Init(sqlite3 *db){

 #ifdef SQLITE_TEST
  if( rc==SQLITE_OK ){
-    rc = sqlite3Fts3ExprInitTestInterface(db);
+    rc = sqlite3Fts3ExprInitTestInterface(db, pHash);
  }
 #endif

@@ -4150,6 +4166,7 @@ static int fts3EvalPhraseLoad(
  return rc;
 }

+#ifndef SQLITE_DISABLE_FTS4_DEFERRED
 /*
 ** This function is called on each phrase after the position lists for
 ** any deferred tokens have been loaded into memory. It updates the phrases
@@ -4253,6 +4270,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){

  return SQLITE_OK;
 }
+#endif /* SQLITE_DISABLE_FTS4_DEFERRED */

 /*
 ** Maximum number of tokens a phrase may have to be considered for the
@@ -584,7 +584,7 @@ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
 );
 void sqlite3Fts3ExprFree(Fts3Expr *);
 #ifdef SQLITE_TEST
-int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
 int sqlite3Fts3InitTerm(sqlite3 *db);
 #endif

@@ -539,7 +539,8 @@ int sqlite3Fts3InitAux(sqlite3 *db){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
  };
  int rc;                         /* Return code */

@@ -1108,34 +1108,6 @@ void sqlite3Fts3ExprFree(Fts3Expr *pDel){

 #include <stdio.h>

-/*
-** Function to query the hash-table of tokenizers (see README.tokenizers).
-*/
-static int queryTestTokenizer(
-  sqlite3 *db, 
-  const char *zName,  
-  const sqlite3_tokenizer_module **pp
-){
-  int rc;
-  sqlite3_stmt *pStmt;
-  const char zSql[] = "SELECT fts3_tokenizer(?)";
-
-  *pp = 0;
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  if( rc!=SQLITE_OK ){
-    return rc;
-  }
-
-  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
-  if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
-      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
-    }
-  }
-
-  return sqlite3_finalize(pStmt);
-}
-
 /*
 ** Return a pointer to a buffer containing a text representation of the
 ** expression passed as the first argument. The buffer is obtained from
@@ -1203,12 +1175,12 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){
 **
 **   SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
 */
-static void fts3ExprTest(
+static void fts3ExprTestCommon(
+  int bRebalance,
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
 ){
-  sqlite3_tokenizer_module const *pModule = 0;
  sqlite3_tokenizer *pTokenizer = 0;
  int rc;
  char **azCol = 0;
@@ -1218,7 +1190,9 @@ static void fts3ExprTest(
  int ii;
  Fts3Expr *pExpr;
  char *zBuf = 0;
-  sqlite3 *db = sqlite3_context_db_handle(context);
+  Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context);
+  const char *zTokenizer = 0;
+  char *zErr = 0;

  if( argc<3 ){
    sqlite3_result_error(context, 
@@ -1227,24 +1201,18 @@ static void fts3ExprTest(
    return;
  }

-  rc = queryTestTokenizer(db,
-                          (const char *)sqlite3_value_text(argv[0]), &pModule);
-  if( rc==SQLITE_NOMEM ){
-    sqlite3_result_error_nomem(context);
-    goto exprtest_out;
-  }else if( !pModule ){
-    sqlite3_result_error(context, "No such tokenizer module", -1);
-    goto exprtest_out;
+  zTokenizer = (const char*)sqlite3_value_text(argv[0]);
+  rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr);
+  if( rc!=SQLITE_OK ){
+    if( rc==SQLITE_NOMEM ){
+      sqlite3_result_error_nomem(context);
+    }else{
+      sqlite3_result_error(context, zErr, -1);
+    }
+    sqlite3_free(zErr);
+    return;
  }

-  rc = pModule->xCreate(0, 0, &pTokenizer);
-  assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
-  if( rc==SQLITE_NOMEM ){
-    sqlite3_result_error_nomem(context);
-    goto exprtest_out;
-  }
-  pTokenizer->pModule = pModule;
-
  zExpr = (const char *)sqlite3_value_text(argv[1]);
  nExpr = sqlite3_value_bytes(argv[1]);
  nCol = argc-2;
@@ -1257,7 +1225,7 @@ static void fts3ExprTest(
    azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
  }

-  if( sqlite3_user_data(context) ){
+  if( bRebalance ){
    char *zDummy = 0;
    rc = sqlite3Fts3ExprParse(
        pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy
@@ -1283,23 +1251,38 @@ static void fts3ExprTest(
  sqlite3Fts3ExprFree(pExpr);

 exprtest_out:
-  if( pModule && pTokenizer ){
-    rc = pModule->xDestroy(pTokenizer);
+  if( pTokenizer ){
+    rc = pTokenizer->pModule->xDestroy(pTokenizer);
  }
  sqlite3_free(azCol);
 }

+static void fts3ExprTest(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts3ExprTestCommon(0, context, argc, argv);
+}
+static void fts3ExprTestRebalance(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts3ExprTestCommon(1, context, argc, argv);
+}
+
 /*
 ** Register the query expression parser test function fts3_exprtest() 
 ** with database connection db. 
 */
-int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){
  int rc = sqlite3_create_function(
-      db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
+      db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0
  );
  if( rc==SQLITE_OK ){
    rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", 
-        -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
+        -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0
    );
  }
  return rc;
@@ -361,7 +361,8 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
  };
  int rc;                         /* Return code */

@@ -443,7 +443,8 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
  };
  int rc;                         /* Return code */

@@ -82,7 +82,7 @@ typedef struct unicode_cursor unicode_cursor;

 struct unicode_tokenizer {
  sqlite3_tokenizer base;
-  int bRemoveDiacritic;
+  int eRemoveDiacritic;
  int nException;
  int *aiException;
 };
@@ -227,17 +227,20 @@ static int unicodeCreate(
  pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
  if( pNew==NULL ) return SQLITE_NOMEM;
  memset(pNew, 0, sizeof(unicode_tokenizer));
-  pNew->bRemoveDiacritic = 1;
+  pNew->eRemoveDiacritic = 1;

  for(i=0; rc==SQLITE_OK && i<nArg; i++){
    const char *z = azArg[i];
    int n = (int)strlen(z);

    if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 1;
+      pNew->eRemoveDiacritic = 1;
    }
    else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 0;
+      pNew->eRemoveDiacritic = 0;
+    }
+    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
+      pNew->eRemoveDiacritic = 2;
    }
    else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
      rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
@@ -350,7 +353,7 @@ static int unicodeNext(

    /* Write the folded case of the last character read to the output */
    zEnd = z;
-    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
+    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
    if( iOut ){
      WRITE_UTF8(zOut, iOut);
    }
@@ -159,32 +159,47 @@ int sqlite3FtsUnicodeIsalnum(int c){
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int remove_diacritic(int c){
+static int remove_diacritic(int c, int bComplex){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
  };
  char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
+    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
+    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
+    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
+    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
+    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
+    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
+    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
+    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
+    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
+    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
+    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
+    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
+    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
+    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
+    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
+    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
+    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
+    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
+    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -201,7 +216,8 @@ static int remove_diacritic(int c){
    }
  }
  assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }


@@ -228,7 +244,7 @@ int sqlite3FtsUnicodeIsdiacritic(int c){
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
@@ -351,7 +367,9 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
      assert( ret>0 );
    }

-    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = remove_diacritic(ret, eRemoveDiacritic==2);
+    }
  }
  
  else if( c>=66560 && c<66600 ){
@@ -1374,15 +1374,19 @@ static int fts3SegReaderNext(
  ** safe (no risk of overread) even if the node data is corrupted. */
  pNext += fts3GetVarint32(pNext, &nPrefix);
  pNext += fts3GetVarint32(pNext, &nSuffix);
-  if( nPrefix<0 || nSuffix<=0 
-   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
+  if( nSuffix<=0 
+   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
+   || nPrefix>pReader->nTermAlloc
  ){
    return FTS_CORRUPT_VTAB;
  }

-  if( nPrefix+nSuffix>pReader->nTermAlloc ){
-    int nNew = (nPrefix+nSuffix)*2;
-    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
+  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
+  ** overflow - hence the (i64) casts.  */
+  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
+    i64 nNew = ((i64)nPrefix+nSuffix)*2;
+    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
@@ -1404,7 +1408,7 @@ static int fts3SegReaderNext(
  ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  ** of these statements is untrue, then the data structure is corrupt.
  */
-  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
+  if( (&pReader->aNode[pReader->nNode] - pReader->aDoclist)<pReader->nDoclist
   || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  ){
    return FTS_CORRUPT_VTAB;
@@ -3730,6 +3734,9 @@ static int nodeReaderNext(NodeReader *p){
    }
    p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);

+    if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
+      return SQLITE_CORRUPT_VTAB;
+    }
    blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
    if( rc==SQLITE_OK ){
      memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
@@ -3737,6 +3744,9 @@ static int nodeReaderNext(NodeReader *p){
      p->iOff += nSuffix;
      if( p->iChild==0 ){
        p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
+        if( (p->nNode-p->iOff)<p->nDoclist ){
+          return SQLITE_CORRUPT_VTAB;
+        }
        p->aDoclist = &p->aNode[p->iOff];
        p->iOff += p->nDoclist;
      }
@@ -3744,7 +3754,6 @@ static int nodeReaderNext(NodeReader *p){
  }

  assert( p->iOff<=p->nNode );
-
  return rc;
 }

@@ -9,11 +9,12 @@ proc print_rd {map} {
  set nRange 1
  set iFirst  [lindex $map 0 0]
  set cPrev   [lindex $map 0 1]
+  set fPrev   [lindex $map 0 2]

  foreach m [lrange $map 1 end] {
-    foreach {i c} $m {}
+    foreach {i c f} $m {}

-    if {$cPrev == $c} {
+    if {$cPrev == $c && $fPrev==$f} {
      for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
        if {[info exists tl_lookup_table($j)]==0} break
      }
@@ -29,13 +30,16 @@ proc print_rd {map} {

    lappend lRange [list $iFirst $nRange]
    lappend aChar  $cPrev
+    lappend aFlag  $fPrev

    set iFirst $i
    set cPrev  $c
+    set fPrev  $f
    set nRange 1
  }
  lappend lRange [list $iFirst $nRange]
  lappend aChar $cPrev
+  lappend aFlag $fPrev

  puts "/*"
  puts "** If the argument is a codepoint corresponding to a lowercase letter"
@@ -45,7 +49,7 @@ proc print_rd {map} {
  puts "** E\"). The resuls of passing a codepoint that corresponds to an"
  puts "** uppercase letter are undefined."
  puts "*/"
-  puts "static int ${::remove_diacritic}(int c)\{"
+  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
  puts "  unsigned short aDia\[\] = \{"
  puts -nonewline "        0, "
  set i 1
@@ -60,13 +64,17 @@ proc print_rd {map} {
  puts ""
  puts "  \};"
  puts "  char aChar\[\] = \{"
-  puts -nonewline "    '\\0', "
+  puts -nonewline "    '\\0',      "
  set i 1
-  foreach c $aChar {
-    set str "'$c',  "
-    if {$c == ""} { set str "'\\0', " }
+  foreach c $aChar f $aFlag {
+    if { $f } {
+      set str "'$c'|0x80,  "
+    } else {
+      set str "'$c'|0x00,  "
+    }
+    if {$c == ""} { set str "'\\0',      " }

-    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
+    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
    incr i
    puts -nonewline "$str"
  }
@@ -87,7 +95,8 @@ proc print_rd {map} {
    }
  }
  assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
  puts "\}"
 }

@@ -95,7 +104,8 @@ proc print_isdiacritic {zFunc map} {

  set lCode [list]
  foreach m $map {
-    foreach {code char} $m {}
+    foreach {code char flag} $m {}
+    if {$flag} continue
    if {$code && $char == ""} { lappend lCode $code }
  }
  set lCode [lsort -integer $lCode]
@@ -472,7 +482,7 @@ proc print_fold {zFunc} {
  puts "** The results are undefined if the value passed to this function"
  puts "** is less than zero."
  puts "*/"
-  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
+  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"

  set liOff [tl_generate_ioff_table $lRecord]
  tl_print_table_header
@@ -516,7 +526,9 @@ proc print_fold {zFunc} {
      assert( ret>0 );
    }

-    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
+    if( eRemoveDiacritic ){
+      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
+    }
  }
  }]

@@ -529,6 +541,259 @@ proc print_fold {zFunc} {
  puts "\}"
 }

+proc code {txt} {
+  set txt [string trimright $txt]
+  set txt [string trimleft $txt "\n"]
+  set n [expr {[string length $txt] - [string length [string trim $txt]]}]
+  set ret ""
+  foreach L [split $txt "\n"] {
+    append ret "[string range $L $n end]\n"
+  }
+  return [uplevel "subst -nocommands {$ret}"]
+}
+
+proc intarray {lInt} {
+  set ret ""
+  set n [llength $lInt]
+  for {set i 0} {$i < $n} {incr i 10} {
+    append ret "\n    "
+    foreach int [lrange $lInt $i [expr $i+9]] {
+      append ret [format "%-7s" "$int, "]
+    }
+  }
+  append ret "\n  "
+  set ret
+}
+
+proc categories_switch {Cvar first lSecond} {
+  upvar $Cvar C
+  set ret ""
+  append ret "case '$first':\n"
+  append ret "          switch( zCat\[1\] ){\n"
+  foreach s $lSecond {
+    append ret "            case '$s': aArray\[$C($first$s)\] = 1; break;\n"
+  }
+  append ret "            case '*': \n"
+  foreach s $lSecond {
+    append ret "              aArray\[$C($first$s)\] = 1;\n"
+  }
+  append ret "              break;\n"
+  append ret "            default: return 1;"
+  append ret "          }\n"
+  append ret "          break;\n"
+}
+
+# Argument is a list. Each element of which is itself a list of two elements:
+#
+#   * the codepoint
+#   * the category
+#
+# List elements are sorted in order of codepoint.
+#
+proc print_categories {lMap} {
+  set categories {
+    Cc Cf Cn Cs
+    Ll Lm Lo Lt Lu
+    Mc Me Mn
+    Nd Nl No
+    Pc Pd Pe Pf Pi Po Ps
+    Sc Sk Sm So
+    Zl Zp Zs
+
+    LC Co
+  }
+
+  for {set i 0} {$i < [llength $categories]} {incr i} {
+    set C([lindex $categories $i]) [expr 1+$i]
+  }
+
+  set caseC [categories_switch C C {c f n s o}]
+  set caseL [categories_switch C L {l m o t u C}]
+  set caseM [categories_switch C M {c e n}]
+  set caseN [categories_switch C N {d l o}]
+  set caseP [categories_switch C P {c d e f i o s}]
+  set caseS [categories_switch C S {c k m o}]
+  set caseZ [categories_switch C Z {l p s}]
+
+  set nCat [expr [llength [array names C]] + 1]
+  puts [code {
+    int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
+      aArray[0] = 1;
+      switch( zCat[0] ){
+        $caseC
+        $caseL
+        $caseM
+        $caseN
+        $caseP
+        $caseS
+        $caseZ
+      }
+      return 0;
+    }
+  }]
+
+  set nRepeat 0
+  set first   [lindex $lMap 0 0]
+  set class   [lindex $lMap 0 1]
+  set prev -1
+
+  set CASE(0) "Lu"
+  set CASE(1) "Ll"
+
+  foreach m $lMap {
+    foreach {codepoint cl} $m {}
+    set codepoint [expr "0x$codepoint"]
+    if {$codepoint>=(1<<20)} continue
+
+    set bNew 0
+    if {$codepoint!=($prev+1)} {
+      set bNew 1
+    } elseif {
+      $cl==$class || ($class=="LC" && $cl==$CASE([expr $nRepeat & 0x01]))
+    } {
+      incr nRepeat
+    } elseif {$class=="Lu" && $nRepeat==1 && $cl=="Ll"} {
+      set class LC
+      incr nRepeat
+    } else {
+      set bNew 1
+    }
+    if {$bNew} {
+      lappend lEntries [list $first $class $nRepeat]
+      set nRepeat 1
+      set first $codepoint
+      set class $cl
+    }
+    set prev $codepoint
+  }
+  if {$nRepeat>0} {
+    lappend lEntries [list $first $class $nRepeat]
+  }
+
+  set aBlock [list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+  set aMap [list]
+  foreach e $lEntries {
+    foreach {cp class nRepeat} $e {}
+    set block [expr ($cp>>16)]
+    if {$block>0 && [lindex $aBlock $block]==0} {
+      for {set i 1} {$i<=$block} {incr i} {
+        if {[lindex $aBlock $i]==0} {
+          lset aBlock $i [llength $aMap]
+        }
+      }
+    }
+    lappend aMap [expr {$cp & 0xFFFF}]
+    lappend aData [expr {($nRepeat << 5) + $C($class)}]
+  }
+  for {set i 1} {$i<[llength $aBlock]} {incr i} {
+    if {[lindex $aBlock $i]==0} {
+      lset aBlock $i [llength $aMap]
+    }
+  }
+
+  set aBlockArray [intarray $aBlock]
+  set aMapArray [intarray $aMap]
+  set aDataArray [intarray $aData]
+  puts [code {
+    static u16 aFts5UnicodeBlock[] = {$aBlockArray};
+    static u16 aFts5UnicodeMap[] = {$aMapArray};
+    static u16 aFts5UnicodeData[] = {$aDataArray};
+
+    int sqlite3Fts5UnicodeCategory(int iCode) { 
+      int iRes = -1;
+      int iHi;
+      int iLo;
+      int ret;
+      u16 iKey;
+
+      if( iCode>=(1<<20) ){
+        return 0;
+      }
+      iLo = aFts5UnicodeBlock[(iCode>>16)];
+      iHi = aFts5UnicodeBlock[1+(iCode>>16)];
+      iKey = (iCode & 0xFFFF);
+      while( iHi>iLo ){
+        int iTest = (iHi + iLo) / 2;
+        assert( iTest>=iLo && iTest<iHi );
+        if( iKey>=aFts5UnicodeMap[iTest] ){
+          iRes = iTest;
+          iLo = iTest+1;
+        }else{
+          iHi = iTest;
+        }
+      }
+
+      if( iRes<0 ) return 0;
+      if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
+      ret = aFts5UnicodeData[iRes] & 0x1F;
+      if( ret!=$C(LC) ) return ret;
+      return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? $C(Ll) : $C(Lu);
+    }
+
+    void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
+      int i = 0;
+      int iTbl = 0;
+      while( i<128 ){
+        int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
+        int n = (aFts5UnicodeData[iTbl] >> 5) + i;
+        for(; i<128 && i<n; i++){
+          aAscii[i] = bToken;
+        }
+        iTbl++;
+      }
+    }
+  }]
+}
+
+proc print_test_categories {lMap} {
+
+  set lCP [list]
+  foreach e $lMap {
+    foreach {cp cat} $e {}
+    if {[expr 0x$cp] < (1<<20)} {
+      lappend lCP "{0x$cp, \"$cat\"}, "
+    }
+  }
+
+  set aCP "\n"
+  for {set i 0} {$i < [llength $lCP]} {incr i 4} {
+    append aCP "    [join [lrange $lCP $i $i+3]]\n"
+  }
+
+
+  puts [code {
+    static int categories_test (int *piCode){
+      struct Codepoint {
+        int iCode;
+        const char *zCat;
+      } aCP[] = {$aCP};
+      int i;
+      int iCP = 0;
+
+      for(i=0; i<1000000; i++){
+        u8 aArray[40];
+        int cat = 0;
+        int c = 0;
+        memset(aArray, 0, sizeof(aArray));
+        if( aCP[iCP].iCode==i ){
+          sqlite3Fts5UnicodeCatParse(aCP[iCP].zCat, aArray);
+          iCP++;
+        }else{
+          aArray[0] = 1;
+        }
+
+        c = sqlite3Fts5UnicodeCategory(i);
+        if( aArray[c]==0 ){
+          *piCode = i;
+          return 1;
+        }
+      }
+
+      return 0;
+    }
+  }]
+}
+
 proc print_fold_test {zFunc mappings} {
  global tl_lookup_table

@@ -605,15 +870,21 @@ proc print_test_main {} {
  puts "#include <stdio.h>"
  puts ""
  puts "int main(int argc, char **argv)\{"
-  puts "  int r1, r2;"
+  puts "  int r1, r2, r3;"
  puts "  int code;"
+  puts "  r3 = 0;"
  puts "  r1 = isalnum_test(&code);"
  puts "  if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);"
  puts "  else printf(\"isalnum(): test passed\\n\");"
  puts "  r2 = fold_test(&code);"
  puts "  if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);"
  puts "  else printf(\"fold(): test passed\\n\");"
-  puts "  return (r1 || r2);"
+  if {$::generate_fts5_code} {
+    puts "  r3 = categories_test(&code);"
+    puts "  if( r3 ) printf(\"categories(): Problem with code %d\\n\",code);"
+    puts "  else printf(\"categories(): test passed\\n\");"
+  }
+  puts "  return (r1 || r2 || r3);"
  puts "\}"
 }

@@ -651,10 +922,18 @@ for {set i 0} {$i < [llength $argv]-2} {incr i} {

 print_fileheader

+if {$::generate_test_code} {
+  puts "typedef unsigned short int u16;"
+  puts "typedef unsigned char u8;"
+  puts "#include <string.h>"
+}
+
 # Print the isalnum() function to stdout.
 #
 set lRange [an_load_separator_ranges]
-print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+if {$generate_fts5_code==0} {
+  print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+}

 # Leave a gap between the two generated C functions.
 #
@@ -677,12 +956,21 @@ puts ""
 #
 print_fold ${function_prefix}UnicodeFold

+if {$generate_fts5_code} {
+  puts ""
+  puts ""
+  print_categories [cc_load_unicodedata_text ${unicodedata.txt}]
+}
+
 # Print the test routines and main() function to stdout, if -test 
 # was specified.
 #
 if {$::generate_test_code} {
-  print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+  if {$generate_fts5_code==0} {
+    print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+  }
  print_fold_test ${function_prefix}UnicodeFold $mappings
+  print_test_categories [cc_load_unicodedata_text ${unicodedata.txt}]
  print_test_main 
 }

@@ -7,12 +7,24 @@
 # character that it should be replaced with, or an empty string if the 
 # codepoint should simply be removed from the input. Examples:
 #
-#   { 224 a  }     (replace codepoint 224 to "a")
-#   { 769 "" }     (remove codepoint 769 from input)
+#   { 224 a  0 }     (replace codepoint 224 to "a")
+#   { 769 "" 0 }     (remove codepoint 769 from input)
 #
 # Mappings are only returned for non-upper case codepoints. It is assumed
 # that the input has already been folded to lower case.
 #
+# The third value in the list is always either 0 or 1. 0 if the 
+# UnicodeData.txt file maps the codepoint to a single ASCII character and
+# a diacritic, or 1 if the mapping is indirect. For example, consider the 
+# two entries:
+#
+# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
+# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
+#
+# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
+# diacritic). The second is an indirect mapping, as it maps to the
+# first codepoint plus 0302 (a diacritic).
+#
 proc rd_load_unicodedata_text {zName} {
  global tl_lookup_table

@@ -53,18 +65,29 @@ proc rd_load_unicodedata_text {zName} {
    set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
    set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]

+    # Filter out upper-case characters, as they will be mapped to their
+    # lower-case equivalents before this data is used.
    if {[info exists tl_lookup_table($iCode)]} continue

+    # Check if this is an indirect mapping. If so, set bIndirect to true
+    # and change $iAscii to the indirectly mappped ASCII character.
+    set bIndirect 0
+    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
+      set iAscii $mapping($iAscii)
+      set bIndirect 1
+    }
+
    if { ($iAscii >= 97 && $iAscii <= 122)
      || ($iAscii >= 65 && $iAscii <= 90)
    } {
-      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
+      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
+      set mapping($iCode) $iAscii
      set dia($iDia) 1
    }
  }

  foreach d [array names dia] {
-    lappend lRet [list $d ""]
+    lappend lRet [list $d "" 0]
  }
  set lRet [lsort -integer -index 0 $lRet]

@@ -143,4 +166,40 @@ proc tl_load_casefolding_txt {zName} {
  }
 }

+proc cc_load_unicodedata_text {zName} {
+  set fd [open $zName]
+  set lField {
+    code
+    character_name
+    general_category
+    canonical_combining_classes
+    bidirectional_category
+    character_decomposition_mapping
+    decimal_digit_value
+    digit_value
+    numeric_value
+    mirrored
+    unicode_1_name
+    iso10646_comment_field
+    uppercase_mapping
+    lowercase_mapping
+    titlecase_mapping
+  }
+  set lRet [list]
+
+  while { ![eof $fd] } {
+    set line [gets $fd]
+    if {$line == ""} continue
+
+    set fields [split $line ";"]
+    if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
+    foreach $lField $fields {}
+
+    lappend lRet [list $code $general_category]
+  }
+
+  close $fd
+  set lRet
+}
+

@@ -444,7 +444,7 @@ struct Fts5ExtensionApi {
 **            This way, even if the tokenizer does not provide synonyms
 **            when tokenizing query text (it should not - to do would be
 **            inefficient), it doesn't matter if the user queries for 
-**            'first + place' or '1st + place', as there are entires in the
+**            'first + place' or '1st + place', as there are entries in the
 **            FTS index corresponding to both forms of the first token.
 **   </ol>
 **
@@ -472,7 +472,7 @@ struct Fts5ExtensionApi {
 **   extra data to the FTS index or require FTS5 to query for multiple terms,
 **   so it is efficient in terms of disk space and query speed. However, it
 **   does not support prefix queries very well. If, as suggested above, the
-**   token "first" is subsituted for "1st" by the tokenizer, then the query:
+**   token "first" is substituted for "1st" by the tokenizer, then the query:
 **
 **   <codeblock>
 **     ... MATCH '1s*'</codeblock>
@@ -784,9 +784,12 @@ int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
 /**************************************************************************
 ** Interface to automatically generated code in fts5_unicode2.c. 
 */
-int sqlite3Fts5UnicodeIsalnum(int c);
 int sqlite3Fts5UnicodeIsdiacritic(int c);
 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
+
+int sqlite3Fts5UnicodeCatParse(const char*, u8*);
+int sqlite3Fts5UnicodeCategory(int iCode);
+void sqlite3Fts5UnicodeAscii(u8*, u8*);
 /*
 ** End of interface to code in fts5_unicode2.c.
 **************************************************************************/
@@ -36,6 +36,7 @@ void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
 #include <stdio.h>
 void sqlite3Fts5ParserTrace(FILE*, char*);
 #endif
+int sqlite3Fts5ParserFallback(int);


 struct Fts5Expr {
@@ -2540,14 +2541,19 @@ static void fts5ExprIsAlnum(
  sqlite3_value **apVal           /* Function arguments */
 ){
  int iCode;
+  u8 aArr[32];
  if( nArg!=1 ){
    sqlite3_result_error(pCtx, 
        "wrong number of arguments to function fts5_isalnum", -1
    );
    return;
  }
+  memset(aArr, 0, sizeof(aArr));
+  sqlite3Fts5UnicodeCatParse("L*", aArr);
+  sqlite3Fts5UnicodeCatParse("N*", aArr);
+  sqlite3Fts5UnicodeCatParse("Co", aArr);
  iCode = sqlite3_value_int(apVal[0]);
-  sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode));
+  sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]);
 }

 static void fts5ExprFold(
@@ -2591,10 +2597,12 @@ int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
    rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
  }

-  /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */
+  /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and
+  ** sqlite3Fts5ParserFallback() are unused */
 #ifndef NDEBUG
  (void)sqlite3Fts5ParserTrace;
 #endif
+  (void)sqlite3Fts5ParserFallback;

  return rc;
 }
@@ -5261,7 +5261,10 @@ int sqlite3Fts5IndexCharlenToBytelen(
  for(i=0; i<nChar; i++){
    if( n>=nByte ) return 0;      /* Input contains fewer than nChar chars */
    if( (unsigned char)p[n++]>=0xc0 ){
-      while( (p[n] & 0xc0)==0x80 ) n++;
+      while( (p[n] & 0xc0)==0x80 ){
+        n++;
+        if( n>=nByte ) break;
+      }
    }
  }
  return n;
@@ -5399,7 +5402,7 @@ int sqlite3Fts5IndexQuery(
      fts5CloseReader(p);
    }

-    *ppIter = &pRet->base;
+    *ppIter = (Fts5IndexIter*)pRet;
    sqlite3Fts5BufferFree(&buf);
  }
  return fts5IndexReturn(p);
@@ -280,7 +280,7 @@ static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){
    case FTS5_SAVEPOINT:
      assert( p->ts.eState==1 );
      assert( iSavepoint>=0 );
-      assert( iSavepoint>p->ts.iSavepoint );
+      assert( iSavepoint>=p->ts.iSavepoint );
      p->ts.iSavepoint = iSavepoint;
      break;
      
@@ -535,6 +535,12 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  aColMap[1] = nCol;
  aColMap[2] = nCol+1;

+  assert( SQLITE_INDEX_CONSTRAINT_EQ<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_GT<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_GE<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
+
  /* Set idxFlags flags for all WHERE clause terms that will be used. */
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
@@ -553,11 +559,11 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
        pInfo->estimatedCost = 1e50;
        return SQLITE_OK;
      }
-    }else{
+    }else if( p->op<=SQLITE_INDEX_CONSTRAINT_MATCH ){
      int j;
      for(j=1; j<ArraySize(aConstraint); j++){
        struct Constraint *pC = &aConstraint[j];
-        if( iCol==aColMap[pC->iCol] && p->op & pC->op && p->usable ){
+        if( iCol==aColMap[pC->iCol] && (p->op & pC->op) && p->usable ){
          pC->iConsIndex = i;
          idxFlags |= pC->fts5op;
        }
@@ -1199,6 +1205,13 @@ static int fts5FilterMethod(
    assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
    assert( pCsr->iLastRowid==LARGEST_INT64 );
    assert( pCsr->iFirstRowid==SMALLEST_INT64 );
+    if( pTab->pSortCsr->bDesc ){
+      pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid;
+      pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid;
+    }else{
+      pCsr->iLastRowid = pTab->pSortCsr->iLastRowid;
+      pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid;
+    }
    pCsr->ePlan = FTS5_PLAN_SOURCE;
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bDesc);
@@ -2632,9 +2645,24 @@ static void fts5SourceIdFunc(
  sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT);
 }

+/*
+** Return true if zName is the extension on one of the shadow tables used
+** by this module.
+*/
+static int fts5ShadowName(const char *zName){
+  static const char *azName[] = {
+    "config", "content", "data", "docsize", "idx"
+  };
+  unsigned int i;
+  for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
+    if( sqlite3_stricmp(zName, azName[i])==0 ) return 1;
+  }
+  return 0;
+}
+
 static int fts5Init(sqlite3 *db){
  static const sqlite3_module fts5Mod = {
-    /* iVersion      */ 2,
+    /* iVersion      */ 3,
    /* xCreate       */ fts5CreateMethod,
    /* xConnect      */ fts5ConnectMethod,
    /* xBestIndex    */ fts5BestIndexMethod,
@@ -2657,6 +2685,7 @@ static int fts5Init(sqlite3 *db){
    /* xSavepoint    */ fts5SavepointMethod,
    /* xRelease      */ fts5ReleaseMethod,
    /* xRollbackTo   */ fts5RollbackToMethod,
+    /* xShadowName   */ fts5ShadowName
  };

  int rc;
@@ -471,7 +471,8 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
  };
  int rc;                         /* Return code */

@@ -234,11 +234,18 @@ struct Unicode61Tokenizer {
  unsigned char aTokenChar[128];  /* ASCII range token characters */
  char *aFold;                    /* Buffer to fold text into */
  int nFold;                      /* Size of aFold[] in bytes */
-  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
+  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
  int nException;
  int *aiException;
+
+  unsigned char aCategory[32];    /* True for token char categories */
 };

+/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
+#define FTS5_REMOVE_DIACRITICS_NONE    0
+#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
+#define FTS5_REMOVE_DIACRITICS_COMPLEX 2
+
 static int fts5UnicodeAddExceptions(
  Unicode61Tokenizer *p,          /* Tokenizer object */
  const char *z,                  /* Characters to treat as exceptions */
@@ -261,7 +268,7 @@ static int fts5UnicodeAddExceptions(
        if( iCode<128 ){
          p->aTokenChar[iCode] = (unsigned char)bTokenChars;
        }else{
-          bToken = sqlite3Fts5UnicodeIsalnum(iCode);
+          bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
          assert( (bToken==0 || bToken==1) ); 
          assert( (bTokenChars==0 || bTokenChars==1) );
          if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
@@ -322,6 +329,21 @@ static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
  return;
 }

+static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
+  const char *z = zCat;
+
+  while( *z ){
+    while( *z==' ' || *z=='\t' ) z++;
+    if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
+      return SQLITE_ERROR;
+    }
+    while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
+  }
+
+  sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
+  return SQLITE_OK;
+}
+
 /*
 ** Create a "unicode61" tokenizer.
 */
@@ -340,32 +362,54 @@ static int fts5UnicodeCreate(
  }else{
    p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
    if( p ){
+      const char *zCat = "L* N* Co";
      int i;
      memset(p, 0, sizeof(Unicode61Tokenizer));
-      memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
-      p->bRemoveDiacritic = 1;
+
+      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
      p->nFold = 64;
      p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
      if( p->aFold==0 ){
        rc = SQLITE_NOMEM;
      }
+
+      /* Search for a "categories" argument */
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
+          zCat = azArg[i+1];
+        }
+      }
+
+      if( rc==SQLITE_OK ){
+        rc = unicodeSetCategories(p, zCat);
+      }
+
      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
        const char *zArg = azArg[i+1];
        if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
+          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
            rc = SQLITE_ERROR;
+          }else{
+            p->eRemoveDiacritic = (zArg[0] - '0');
+            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
+                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
+                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
+            );
          }
-          p->bRemoveDiacritic = (zArg[0]=='1');
        }else
        if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
          rc = fts5UnicodeAddExceptions(p, zArg, 1);
        }else
        if( 0==sqlite3_stricmp(azArg[i], "separators") ){
          rc = fts5UnicodeAddExceptions(p, zArg, 0);
+        }else
+        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
+          /* no-op */
        }else{
          rc = SQLITE_ERROR;
        }
      }
+
    }else{
      rc = SQLITE_NOMEM;
    }
@@ -384,8 +428,10 @@ static int fts5UnicodeCreate(
 ** character (not a separator).
 */
 static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
-  assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
-  return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode);
+  return (
+    p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]
+    ^ fts5UnicodeIsException(p, iCode)
+  );
 }

 static int fts5UnicodeTokenize(
@@ -463,7 +509,7 @@ static int fts5UnicodeTokenize(
        READ_UTF8(zCsr, zTerm, iCode);
        if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
 non_ascii_tokenchar:
-          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
+          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
          if( iCode ) WRITE_UTF8(zOut, iCode);
        }else{
          break;
@@ -18,135 +18,6 @@

 #include <assert.h>

-/*
-** Return true if the argument corresponds to a unicode codepoint
-** classified as either a letter or a number. Otherwise false.
-**
-** The results are undefined if the value passed to this function
-** is less than zero.
-*/
-int sqlite3Fts5UnicodeIsalnum(int c){
-  /* Each unsigned integer in the following array corresponds to a contiguous
-  ** range of unicode codepoints that are not either letters or numbers (i.e.
-  ** codepoints for which this function should return 0).
-  **
-  ** The most significant 22 bits in each 32-bit value contain the first 
-  ** codepoint in the range. The least significant 10 bits are used to store
-  ** the size of the range (always at least 1). In other words, the value 
-  ** ((C<<22) + N) represents a range of N codepoints starting with codepoint 
-  ** C. It is not possible to represent a range larger than 1023 codepoints 
-  ** using this format.
-  */
-  static const unsigned int aEntry[] = {
-    0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
-    0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
-    0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
-    0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
-    0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
-    0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
-    0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
-    0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
-    0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
-    0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
-    0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
-    0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
-    0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
-    0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
-    0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
-    0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
-    0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
-    0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
-    0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
-    0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
-    0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
-    0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
-    0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
-    0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
-    0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
-    0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
-    0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
-    0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
-    0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
-    0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
-    0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
-    0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
-    0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
-    0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
-    0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
-    0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
-    0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
-    0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
-    0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
-    0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
-    0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
-    0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
-    0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
-    0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
-    0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
-    0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
-    0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
-    0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
-    0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
-    0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
-    0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E,
-    0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100,
-    0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10,
-    0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402,
-    0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804,
-    0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012,
-    0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004,
-    0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002,
-    0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
-    0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
-    0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
-    0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
-    0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
-    0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
-    0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
-    0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
-    0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
-    0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
-    0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
-    0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
-    0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
-    0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
-    0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
-    0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
-    0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
-    0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
-    0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
-    0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
-    0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
-    0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
-    0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
-    0x380400F0,
-  };
-  static const unsigned int aAscii[4] = {
-    0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
-  };
-
-  if( (unsigned int)c<128 ){
-    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
-  }else if( (unsigned int)c<(1<<22) ){
-    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
-    int iRes = 0;
-    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
-    int iLo = 0;
-    while( iHi>=iLo ){
-      int iTest = (iHi + iLo) / 2;
-      if( key >= aEntry[iTest] ){
-        iRes = iTest;
-        iLo = iTest+1;
-      }else{
-        iHi = iTest-1;
-      }
-    }
-    assert( aEntry[0]<key );
-    assert( key>=aEntry[iRes] );
-    return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
-  }
-  return 1;
-}


 /*
@@ -157,32 +28,47 @@ int sqlite3Fts5UnicodeIsalnum(int c){
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int fts5_remove_diacritic(int c){
+static int fts5_remove_diacritic(int c, int bComplex){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
  };
  char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
+    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
+    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
+    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
+    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
+    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
+    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
+    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
+    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
+    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
+    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
+    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
+    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
+    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
+    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
+    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
+    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
+    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
+    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
+    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -199,7 +85,8 @@ static int fts5_remove_diacritic(int c){
    }
  }
  assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }


@@ -226,7 +113,7 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
@@ -349,7 +236,9 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
      assert( ret>0 );
    }

-    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
+    }
  }
  
  else if( c>=66560 && c<66600 ){
@@ -358,3 +247,528 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){

  return ret;
 }
+
+int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
+  aArray[0] = 1;
+  switch( zCat[0] ){
+    case 'C':
+          switch( zCat[1] ){
+            case 'c': aArray[1] = 1; break;
+            case 'f': aArray[2] = 1; break;
+            case 'n': aArray[3] = 1; break;
+            case 's': aArray[4] = 1; break;
+            case 'o': aArray[31] = 1; break;
+            case '*': 
+              aArray[1] = 1;
+              aArray[2] = 1;
+              aArray[3] = 1;
+              aArray[4] = 1;
+              aArray[31] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'L':
+          switch( zCat[1] ){
+            case 'l': aArray[5] = 1; break;
+            case 'm': aArray[6] = 1; break;
+            case 'o': aArray[7] = 1; break;
+            case 't': aArray[8] = 1; break;
+            case 'u': aArray[9] = 1; break;
+            case 'C': aArray[30] = 1; break;
+            case '*': 
+              aArray[5] = 1;
+              aArray[6] = 1;
+              aArray[7] = 1;
+              aArray[8] = 1;
+              aArray[9] = 1;
+              aArray[30] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'M':
+          switch( zCat[1] ){
+            case 'c': aArray[10] = 1; break;
+            case 'e': aArray[11] = 1; break;
+            case 'n': aArray[12] = 1; break;
+            case '*': 
+              aArray[10] = 1;
+              aArray[11] = 1;
+              aArray[12] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'N':
+          switch( zCat[1] ){
+            case 'd': aArray[13] = 1; break;
+            case 'l': aArray[14] = 1; break;
+            case 'o': aArray[15] = 1; break;
+            case '*': 
+              aArray[13] = 1;
+              aArray[14] = 1;
+              aArray[15] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'P':
+          switch( zCat[1] ){
+            case 'c': aArray[16] = 1; break;
+            case 'd': aArray[17] = 1; break;
+            case 'e': aArray[18] = 1; break;
+            case 'f': aArray[19] = 1; break;
+            case 'i': aArray[20] = 1; break;
+            case 'o': aArray[21] = 1; break;
+            case 's': aArray[22] = 1; break;
+            case '*': 
+              aArray[16] = 1;
+              aArray[17] = 1;
+              aArray[18] = 1;
+              aArray[19] = 1;
+              aArray[20] = 1;
+              aArray[21] = 1;
+              aArray[22] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'S':
+          switch( zCat[1] ){
+            case 'c': aArray[23] = 1; break;
+            case 'k': aArray[24] = 1; break;
+            case 'm': aArray[25] = 1; break;
+            case 'o': aArray[26] = 1; break;
+            case '*': 
+              aArray[23] = 1;
+              aArray[24] = 1;
+              aArray[25] = 1;
+              aArray[26] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'Z':
+          switch( zCat[1] ){
+            case 'l': aArray[27] = 1; break;
+            case 'p': aArray[28] = 1; break;
+            case 's': aArray[29] = 1; break;
+            case '*': 
+              aArray[27] = 1;
+              aArray[28] = 1;
+              aArray[29] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+  }
+  return 0;
+}
+
+static u16 aFts5UnicodeBlock[] = {
+    0,     1471,  1753,  1760,  1760,  1760,  1760,  1760,  1760,  1760,  
+    1760,  1760,  1760,  1760,  1760,  1763,  1765,  
+  };
+static u16 aFts5UnicodeMap[] = {
+    0,     32,    33,    36,    37,    40,    41,    42,    43,    44,    
+    45,    46,    48,    58,    60,    63,    65,    91,    92,    93,    
+    94,    95,    96,    97,    123,   124,   125,   126,   127,   160,   
+    161,   162,   166,   167,   168,   169,   170,   171,   172,   173,   
+    174,   175,   176,   177,   178,   180,   181,   182,   184,   185,   
+    186,   187,   188,   191,   192,   215,   216,   223,   247,   248,   
+    256,   312,   313,   329,   330,   377,   383,   385,   387,   388,   
+    391,   394,   396,   398,   402,   403,   405,   406,   409,   412,   
+    414,   415,   417,   418,   423,   427,   428,   431,   434,   436,   
+    437,   440,   442,   443,   444,   446,   448,   452,   453,   454,   
+    455,   456,   457,   458,   459,   460,   461,   477,   478,   496,   
+    497,   498,   499,   500,   503,   505,   506,   564,   570,   572,   
+    573,   575,   577,   580,   583,   584,   592,   660,   661,   688,   
+    706,   710,   722,   736,   741,   748,   749,   750,   751,   768,   
+    880,   884,   885,   886,   890,   891,   894,   900,   902,   903,   
+    904,   908,   910,   912,   913,   931,   940,   975,   977,   978,   
+    981,   984,   1008,  1012,  1014,  1015,  1018,  1020,  1021,  1072,  
+    1120,  1154,  1155,  1160,  1162,  1217,  1231,  1232,  1329,  1369,  
+    1370,  1377,  1417,  1418,  1423,  1425,  1470,  1471,  1472,  1473,  
+    1475,  1476,  1478,  1479,  1488,  1520,  1523,  1536,  1542,  1545,  
+    1547,  1548,  1550,  1552,  1563,  1566,  1568,  1600,  1601,  1611,  
+    1632,  1642,  1646,  1648,  1649,  1748,  1749,  1750,  1757,  1758,  
+    1759,  1765,  1767,  1769,  1770,  1774,  1776,  1786,  1789,  1791,  
+    1792,  1807,  1808,  1809,  1810,  1840,  1869,  1958,  1969,  1984,  
+    1994,  2027,  2036,  2038,  2039,  2042,  2048,  2070,  2074,  2075,  
+    2084,  2085,  2088,  2089,  2096,  2112,  2137,  2142,  2208,  2210,  
+    2276,  2304,  2307,  2308,  2362,  2363,  2364,  2365,  2366,  2369,  
+    2377,  2381,  2382,  2384,  2385,  2392,  2402,  2404,  2406,  2416,  
+    2417,  2418,  2425,  2433,  2434,  2437,  2447,  2451,  2474,  2482,  
+    2486,  2492,  2493,  2494,  2497,  2503,  2507,  2509,  2510,  2519,  
+    2524,  2527,  2530,  2534,  2544,  2546,  2548,  2554,  2555,  2561,  
+    2563,  2565,  2575,  2579,  2602,  2610,  2613,  2616,  2620,  2622,  
+    2625,  2631,  2635,  2641,  2649,  2654,  2662,  2672,  2674,  2677,  
+    2689,  2691,  2693,  2703,  2707,  2730,  2738,  2741,  2748,  2749,  
+    2750,  2753,  2759,  2761,  2763,  2765,  2768,  2784,  2786,  2790,  
+    2800,  2801,  2817,  2818,  2821,  2831,  2835,  2858,  2866,  2869,  
+    2876,  2877,  2878,  2879,  2880,  2881,  2887,  2891,  2893,  2902,  
+    2903,  2908,  2911,  2914,  2918,  2928,  2929,  2930,  2946,  2947,  
+    2949,  2958,  2962,  2969,  2972,  2974,  2979,  2984,  2990,  3006,  
+    3008,  3009,  3014,  3018,  3021,  3024,  3031,  3046,  3056,  3059,  
+    3065,  3066,  3073,  3077,  3086,  3090,  3114,  3125,  3133,  3134,  
+    3137,  3142,  3146,  3157,  3160,  3168,  3170,  3174,  3192,  3199,  
+    3202,  3205,  3214,  3218,  3242,  3253,  3260,  3261,  3262,  3263,  
+    3264,  3270,  3271,  3274,  3276,  3285,  3294,  3296,  3298,  3302,  
+    3313,  3330,  3333,  3342,  3346,  3389,  3390,  3393,  3398,  3402,  
+    3405,  3406,  3415,  3424,  3426,  3430,  3440,  3449,  3450,  3458,  
+    3461,  3482,  3507,  3517,  3520,  3530,  3535,  3538,  3542,  3544,  
+    3570,  3572,  3585,  3633,  3634,  3636,  3647,  3648,  3654,  3655,  
+    3663,  3664,  3674,  3713,  3716,  3719,  3722,  3725,  3732,  3737,  
+    3745,  3749,  3751,  3754,  3757,  3761,  3762,  3764,  3771,  3773,  
+    3776,  3782,  3784,  3792,  3804,  3840,  3841,  3844,  3859,  3860,  
+    3861,  3864,  3866,  3872,  3882,  3892,  3893,  3894,  3895,  3896,  
+    3897,  3898,  3899,  3900,  3901,  3902,  3904,  3913,  3953,  3967,  
+    3968,  3973,  3974,  3976,  3981,  3993,  4030,  4038,  4039,  4046,  
+    4048,  4053,  4057,  4096,  4139,  4141,  4145,  4146,  4152,  4153,  
+    4155,  4157,  4159,  4160,  4170,  4176,  4182,  4184,  4186,  4190,  
+    4193,  4194,  4197,  4199,  4206,  4209,  4213,  4226,  4227,  4229,  
+    4231,  4237,  4238,  4239,  4240,  4250,  4253,  4254,  4256,  4295,  
+    4301,  4304,  4347,  4348,  4349,  4682,  4688,  4696,  4698,  4704,  
+    4746,  4752,  4786,  4792,  4800,  4802,  4808,  4824,  4882,  4888,  
+    4957,  4960,  4969,  4992,  5008,  5024,  5120,  5121,  5741,  5743,  
+    5760,  5761,  5787,  5788,  5792,  5867,  5870,  5888,  5902,  5906,  
+    5920,  5938,  5941,  5952,  5970,  5984,  5998,  6002,  6016,  6068,  
+    6070,  6071,  6078,  6086,  6087,  6089,  6100,  6103,  6104,  6107,  
+    6108,  6109,  6112,  6128,  6144,  6150,  6151,  6155,  6158,  6160,  
+    6176,  6211,  6212,  6272,  6313,  6314,  6320,  6400,  6432,  6435,  
+    6439,  6441,  6448,  6450,  6451,  6457,  6464,  6468,  6470,  6480,  
+    6512,  6528,  6576,  6593,  6600,  6608,  6618,  6622,  6656,  6679,  
+    6681,  6686,  6688,  6741,  6742,  6743,  6744,  6752,  6753,  6754,  
+    6755,  6757,  6765,  6771,  6783,  6784,  6800,  6816,  6823,  6824,  
+    6912,  6916,  6917,  6964,  6965,  6966,  6971,  6972,  6973,  6978,  
+    6979,  6981,  6992,  7002,  7009,  7019,  7028,  7040,  7042,  7043,  
+    7073,  7074,  7078,  7080,  7082,  7083,  7084,  7086,  7088,  7098,  
+    7142,  7143,  7144,  7146,  7149,  7150,  7151,  7154,  7164,  7168,  
+    7204,  7212,  7220,  7222,  7227,  7232,  7245,  7248,  7258,  7288,  
+    7294,  7360,  7376,  7379,  7380,  7393,  7394,  7401,  7405,  7406,  
+    7410,  7412,  7413,  7424,  7468,  7531,  7544,  7545,  7579,  7616,  
+    7676,  7680,  7830,  7838,  7936,  7944,  7952,  7960,  7968,  7976,  
+    7984,  7992,  8000,  8008,  8016,  8025,  8027,  8029,  8031,  8033,  
+    8040,  8048,  8064,  8072,  8080,  8088,  8096,  8104,  8112,  8118,  
+    8120,  8124,  8125,  8126,  8127,  8130,  8134,  8136,  8140,  8141,  
+    8144,  8150,  8152,  8157,  8160,  8168,  8173,  8178,  8182,  8184,  
+    8188,  8189,  8192,  8203,  8208,  8214,  8216,  8217,  8218,  8219,  
+    8221,  8222,  8223,  8224,  8232,  8233,  8234,  8239,  8240,  8249,  
+    8250,  8251,  8255,  8257,  8260,  8261,  8262,  8263,  8274,  8275,  
+    8276,  8277,  8287,  8288,  8298,  8304,  8305,  8308,  8314,  8317,  
+    8318,  8319,  8320,  8330,  8333,  8334,  8336,  8352,  8400,  8413,  
+    8417,  8418,  8421,  8448,  8450,  8451,  8455,  8456,  8458,  8459,  
+    8462,  8464,  8467,  8468,  8469,  8470,  8472,  8473,  8478,  8484,  
+    8485,  8486,  8487,  8488,  8489,  8490,  8494,  8495,  8496,  8500,  
+    8501,  8505,  8506,  8508,  8510,  8512,  8517,  8519,  8522,  8523,  
+    8524,  8526,  8527,  8528,  8544,  8579,  8581,  8585,  8592,  8597,  
+    8602,  8604,  8608,  8609,  8611,  8612,  8614,  8615,  8622,  8623,  
+    8654,  8656,  8658,  8659,  8660,  8661,  8692,  8960,  8968,  8972,  
+    8992,  8994,  9001,  9002,  9003,  9084,  9085,  9115,  9140,  9180,  
+    9186,  9216,  9280,  9312,  9372,  9450,  9472,  9655,  9656,  9665,  
+    9666,  9720,  9728,  9839,  9840,  9985,  10088, 10089, 10090, 10091, 
+    10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, 
+    10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217, 
+    10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627, 
+    10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637, 
+    10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, 
+    10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750, 
+    11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365, 
+    11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393, 
+    11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520, 
+    11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696, 
+    11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780, 
+    11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800, 
+    11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812, 
+    11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904, 
+    11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296, 
+    12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306, 
+    12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317, 
+    12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347, 
+    12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449, 
+    12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736, 
+    12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938, 
+    12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981, 
+    40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528, 
+    42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624, 
+    42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800, 
+    42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912, 
+    43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043, 
+    43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136, 
+    43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264, 
+    43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395, 
+    43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472, 
+    43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588, 
+    43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643, 
+    43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713, 
+    43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762, 
+    43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003, 
+    44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203, 
+    55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112, 
+    64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320, 
+    64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020, 
+    65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075, 
+    65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086, 
+    65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097, 
+    65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118, 
+    65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279, 
+    65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294, 
+    65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343, 
+    65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378, 
+    65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490, 
+    65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529, 
+    65532, 0,     13,    40,    60,    63,    80,    128,   256,   263,   
+    311,   320,   373,   377,   394,   400,   464,   509,   640,   672,   
+    768,   800,   816,   833,   834,   842,   896,   927,   928,   968,   
+    976,   977,   1024,  1064,  1104,  1184,  2048,  2056,  2058,  2103,  
+    2108,  2111,  2135,  2136,  2304,  2326,  2335,  2336,  2367,  2432,  
+    2494,  2560,  2561,  2565,  2572,  2576,  2581,  2585,  2616,  2623,  
+    2624,  2640,  2656,  2685,  2687,  2816,  2873,  2880,  2904,  2912,  
+    2936,  3072,  3680,  4096,  4097,  4098,  4099,  4152,  4167,  4178,  
+    4198,  4224,  4226,  4227,  4272,  4275,  4279,  4281,  4283,  4285,  
+    4286,  4304,  4336,  4352,  4355,  4391,  4396,  4397,  4406,  4416,  
+    4480,  4482,  4483,  4531,  4534,  4543,  4545,  4549,  4560,  5760,  
+    5803,  5804,  5805,  5806,  5808,  5814,  5815,  5824,  8192,  9216,  
+    9328,  12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248, 
+    53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637, 
+    53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298, 
+    54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441, 
+    54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541, 
+    54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662, 
+    54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922, 
+    54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062, 
+    55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178, 
+    55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961, 
+    60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003, 
+    61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028, 
+    61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099, 
+    61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744, 
+    61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368, 
+    62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971, 
+    63045, 63104, 63232, 0,     42710, 42752, 46900, 46912, 47133, 63488, 
+    1,     32,    256,   0,     65533, 
+  };
+static u16 aFts5UnicodeData[] = {
+    1025,  61,    117,   55,    117,   54,    50,    53,    57,    53,    
+    49,    85,    333,   85,    121,   85,    841,   54,    53,    50,    
+    56,    48,    56,    837,   54,    57,    50,    57,    1057,  61,    
+    53,    151,   58,    53,    56,    58,    39,    52,    57,    34,    
+    58,    56,    58,    57,    79,    56,    37,    85,    56,    47,    
+    39,    51,    111,   53,    745,   57,    233,   773,   57,    261,   
+    1822,  37,    542,   37,    1534,  222,   69,    73,    37,    126,   
+    126,   73,    69,    137,   37,    73,    37,    105,   101,   73,    
+    37,    73,    37,    190,   158,   37,    126,   126,   73,    37,    
+    126,   94,    37,    39,    94,    69,    135,   41,    40,    37,    
+    41,    40,    37,    41,    40,    37,    542,   37,    606,   37,    
+    41,    40,    37,    126,   73,    37,    1886,  197,   73,    37,    
+    73,    69,    126,   105,   37,    286,   2181,  39,    869,   582,   
+    152,   390,   472,   166,   248,   38,    56,    38,    568,   3596,  
+    158,   38,    56,    94,    38,    101,   53,    88,    41,    53,    
+    105,   41,    73,    37,    553,   297,   1125,  94,    37,    105,   
+    101,   798,   133,   94,    57,    126,   94,    37,    1641,  1541,  
+    1118,  58,    172,   75,    1790,  478,   37,    2846,  1225,  38,    
+    213,   1253,  53,    49,    55,    1452,  49,    44,    53,    76,    
+    53,    76,    53,    44,    871,   103,   85,    162,   121,   85,    
+    55,    85,    90,    364,   53,    85,    1031,  38,    327,   684,   
+    333,   149,   71,    44,    3175,  53,    39,    236,   34,    58,    
+    204,   70,    76,    58,    140,   71,    333,   103,   90,    39,    
+    469,   34,    39,    44,    967,   876,   2855,  364,   39,    333,   
+    1063,  300,   70,    58,    117,   38,    711,   140,   38,    300,   
+    38,    108,   38,    172,   501,   807,   108,   53,    39,    359,   
+    876,   108,   42,    1735,  44,    42,    44,    39,    106,   268,   
+    138,   44,    74,    39,    236,   327,   76,    85,    333,   53,    
+    38,    199,   231,   44,    74,    263,   71,    711,   231,   39,    
+    135,   44,    39,    106,   140,   74,    74,    44,    39,    42,    
+    71,    103,   76,    333,   71,    87,    207,   58,    55,    76,    
+    42,    199,   71,    711,   231,   71,    71,    71,    44,    106,   
+    76,    76,    108,   44,    135,   39,    333,   76,    103,   44,    
+    76,    42,    295,   103,   711,   231,   71,    167,   44,    39,    
+    106,   172,   76,    42,    74,    44,    39,    71,    76,    333,   
+    53,    55,    44,    74,    263,   71,    711,   231,   71,    167,   
+    44,    39,    42,    44,    42,    140,   74,    74,    44,    44,    
+    42,    71,    103,   76,    333,   58,    39,    207,   44,    39,    
+    199,   103,   135,   71,    39,    71,    71,    103,   391,   74,    
+    44,    74,    106,   106,   44,    39,    42,    333,   111,   218,   
+    55,    58,    106,   263,   103,   743,   327,   167,   39,    108,   
+    138,   108,   140,   76,    71,    71,    76,    333,   239,   58,    
+    74,    263,   103,   743,   327,   167,   44,    39,    42,    44,    
+    170,   44,    74,    74,    76,    74,    39,    71,    76,    333,   
+    71,    74,    263,   103,   1319,  39,    106,   140,   106,   106,   
+    44,    39,    42,    71,    76,    333,   207,   58,    199,   74,    
+    583,   775,   295,   39,    231,   44,    106,   108,   44,    266,   
+    74,    53,    1543,  44,    71,    236,   55,    199,   38,    268,   
+    53,    333,   85,    71,    39,    71,    39,    39,    135,   231,   
+    103,   39,    39,    71,    135,   44,    71,    204,   76,    39,    
+    167,   38,    204,   333,   135,   39,    122,   501,   58,    53,    
+    122,   76,    218,   333,   335,   58,    44,    58,    44,    58,    
+    44,    54,    50,    54,    50,    74,    263,   1159,  460,   42,    
+    172,   53,    76,    167,   364,   1164,  282,   44,    218,   90,    
+    181,   154,   85,    1383,  74,    140,   42,    204,   42,    76,    
+    74,    76,    39,    333,   213,   199,   74,    76,    135,   108,   
+    39,    106,   71,    234,   103,   140,   423,   44,    74,    76,    
+    202,   44,    39,    42,    333,   106,   44,    90,    1225,  41,    
+    41,    1383,  53,    38,    10631, 135,   231,   39,    135,   1319,  
+    135,   1063,  135,   231,   39,    135,   487,   1831,  135,   2151,  
+    108,   309,   655,   519,   346,   2727,  49,    19847, 85,    551,   
+    61,    839,   54,    50,    2407,  117,   110,   423,   135,   108,   
+    583,   108,   85,    583,   76,    423,   103,   76,    1671,  76,    
+    42,    236,   266,   44,    74,    364,   117,   38,    117,   55,    
+    39,    44,    333,   335,   213,   49,    149,   108,   61,    333,   
+    1127,  38,    1671,  1319,  44,    39,    2247,  935,   108,   138,   
+    76,    106,   74,    44,    202,   108,   58,    85,    333,   967,   
+    167,   1415,  554,   231,   74,    333,   47,    1114,  743,   76,    
+    106,   85,    1703,  42,    44,    42,    236,   44,    42,    44,    
+    74,    268,   202,   332,   44,    333,   333,   245,   38,    213,   
+    140,   42,    1511,  44,    42,    172,   42,    44,    170,   44,    
+    74,    231,   333,   245,   346,   300,   314,   76,    42,    967,   
+    42,    140,   74,    76,    42,    44,    74,    71,    333,   1415,  
+    44,    42,    76,    106,   44,    42,    108,   74,    149,   1159,  
+    266,   268,   74,    76,    181,   333,   103,   333,   967,   198,   
+    85,    277,   108,   53,    428,   42,    236,   135,   44,    135,   
+    74,    44,    71,    1413,  2022,  421,   38,    1093,  1190,  1260,  
+    140,   4830,  261,   3166,  261,   265,   197,   201,   261,   265,   
+    261,   265,   197,   201,   261,   41,    41,    41,    94,    229,   
+    265,   453,   261,   264,   261,   264,   261,   264,   165,   69,    
+    137,   40,    56,    37,    120,   101,   69,    137,   40,    120,   
+    133,   69,    137,   120,   261,   169,   120,   101,   69,    137,   
+    40,    88,    381,   162,   209,   85,    52,    51,    54,    84,    
+    51,    54,    52,    277,   59,    60,    162,   61,    309,   52,    
+    51,    149,   80,    117,   57,    54,    50,    373,   57,    53,    
+    48,    341,   61,    162,   194,   47,    38,    207,   121,   54,    
+    50,    38,    335,   121,   54,    50,    422,   855,   428,   139,   
+    44,    107,   396,   90,    41,    154,   41,    90,    37,    105,   
+    69,    105,   37,    58,    41,    90,    57,    169,   218,   41,    
+    58,    41,    58,    41,    58,    137,   58,    37,    137,   37,    
+    135,   37,    90,    69,    73,    185,   94,    101,   58,    57,    
+    90,    37,    58,    527,   1134,  94,    142,   47,    185,   186,   
+    89,    154,   57,    90,    57,    90,    57,    250,   57,    1018,  
+    89,    90,    57,    58,    57,    1018,  8601,  282,   153,   666,   
+    89,    250,   54,    50,    2618,  57,    986,   825,   1306,  217,   
+    602,   1274,  378,   1935,  2522,  719,   5882,  57,    314,   57,    
+    1754,  281,   3578,  57,    4634,  3322,  54,    50,    54,    50,    
+    54,    50,    54,    50,    54,    50,    54,    50,    54,    50,    
+    975,   1434,  185,   54,    50,    1017,  54,    50,    54,    50,    
+    54,    50,    54,    50,    54,    50,    537,   8218,  4217,  54,    
+    50,    54,    50,    54,    50,    54,    50,    54,    50,    54,    
+    50,    54,    50,    54,    50,    54,    50,    54,    50,    54,    
+    50,    2041,  54,    50,    54,    50,    1049,  54,    50,    8281,  
+    1562,  697,   90,    217,   346,   1513,  1509,  126,   73,    69,    
+    254,   105,   37,    94,    37,    94,    165,   70,    105,   37,    
+    3166,  37,    218,   158,   108,   94,    149,   47,    85,    1221,  
+    37,    37,    1799,  38,    53,    44,    743,   231,   231,   231,   
+    231,   231,   231,   231,   231,   1036,  85,    52,    51,    52,    
+    51,    117,   52,    51,    53,    52,    51,    309,   49,    85,    
+    49,    53,    52,    51,    85,    52,    51,    54,    50,    54,    
+    50,    54,    50,    54,    50,    181,   38,    341,   81,    858,   
+    2874,  6874,  410,   61,    117,   58,    38,    39,    46,    54,    
+    50,    54,    50,    54,    50,    54,    50,    54,    50,    90,    
+    54,    50,    54,    50,    54,    50,    54,    50,    49,    54,    
+    82,    58,    302,   140,   74,    49,    166,   90,    110,   38,    
+    39,    53,    90,    2759,  76,    88,    70,    39,    49,    2887,  
+    53,    102,   39,    1319,  3015,  90,    143,   346,   871,   1178,  
+    519,   1018,  335,   986,   271,   58,    495,   1050,  335,   1274,  
+    495,   2042,  8218,  39,    39,    2074,  39,    39,    679,   38,    
+    36583, 1786,  1287,  198,   85,    8583,  38,    117,   519,   333,   
+    71,    1502,  39,    44,    107,   53,    332,   53,    38,    798,   
+    44,    2247,  334,   76,    213,   760,   294,   88,    478,   69,    
+    2014,  38,    261,   190,   350,   38,    88,    158,   158,   382,   
+    70,    37,    231,   44,    103,   44,    135,   44,    743,   74,    
+    76,    42,    154,   207,   90,    55,    58,    1671,  149,   74,    
+    1607,  522,   44,    85,    333,   588,   199,   117,   39,    333,   
+    903,   268,   85,    743,   364,   74,    53,    935,   108,   42,    
+    1511,  44,    74,    140,   74,    44,    138,   437,   38,    333,   
+    85,    1319,  204,   74,    76,    74,    76,    103,   44,    263,   
+    44,    42,    333,   149,   519,   38,    199,   122,   39,    42,    
+    1543,  44,    39,    108,   71,    76,    167,   76,    39,    44,    
+    39,    71,    38,    85,    359,   42,    76,    74,    85,    39,    
+    70,    42,    44,    199,   199,   199,   231,   231,   1127,  74,    
+    44,    74,    44,    74,    53,    42,    44,    333,   39,    39,    
+    743,   1575,  36,    68,    68,    36,    63,    63,    11719, 3399,  
+    229,   165,   39,    44,    327,   57,    423,   167,   39,    71,    
+    71,    3463,  536,   11623, 54,    50,    2055,  1735,  391,   55,    
+    58,    524,   245,   54,    50,    53,    236,   53,    81,    80,    
+    54,    50,    54,    50,    54,    50,    54,    50,    54,    50,    
+    54,    50,    54,    50,    54,    50,    85,    54,    50,    149,   
+    112,   117,   149,   49,    54,    50,    54,    50,    54,    50,    
+    117,   57,    49,    121,   53,    55,    85,    167,   4327,  34,    
+    117,   55,    117,   54,    50,    53,    57,    53,    49,    85,    
+    333,   85,    121,   85,    841,   54,    53,    50,    56,    48,    
+    56,    837,   54,    57,    50,    57,    54,    50,    53,    54,    
+    50,    85,    327,   38,    1447,  70,    999,   199,   199,   199,   
+    103,   87,    57,    56,    58,    87,    58,    153,   90,    98,    
+    90,    391,   839,   615,   71,    487,   455,   3943,  117,   1455,  
+    314,   1710,  143,   570,   47,    410,   1466,  44,    935,   1575,  
+    999,   143,   551,   46,    263,   46,    967,   53,    1159,  263,   
+    53,    174,   1289,  1285,  2503,  333,   199,   39,    1415,  71,    
+    39,    743,   53,    271,   711,   207,   53,    839,   53,    1799,  
+    71,    39,    108,   76,    140,   135,   103,   871,   108,   44,    
+    271,   309,   935,   79,    53,    1735,  245,   711,   271,   615,   
+    271,   2343,  1007,  42,    44,    42,    1703,  492,   245,   655,   
+    333,   76,    42,    1447,  106,   140,   74,    76,    85,    34,    
+    149,   807,   333,   108,   1159,  172,   42,    268,   333,   149,   
+    76,    42,    1543,  106,   300,   74,    135,   149,   333,   1383,  
+    44,    42,    44,    74,    204,   42,    44,    333,   28135, 3182,  
+    149,   34279, 18215, 2215,  39,    1482,  140,   422,   71,    7898,  
+    1274,  1946,  74,    108,   122,   202,   258,   268,   90,    236,   
+    986,   140,   1562,  2138,  108,   58,    2810,  591,   841,   837,   
+    841,   229,   581,   841,   837,   41,    73,    41,    73,    137,   
+    265,   133,   37,    229,   357,   841,   837,   73,    137,   265,   
+    233,   837,   73,    137,   169,   41,    233,   837,   841,   837,   
+    841,   837,   841,   837,   841,   837,   841,   837,   841,   901,   
+    809,   57,    805,   57,    197,   809,   57,    805,   57,    197,   
+    809,   57,    805,   57,    197,   809,   57,    805,   57,    197,   
+    809,   57,    805,   57,    197,   94,    1613,  135,   871,   71,    
+    39,    39,    327,   135,   39,    39,    39,    39,    39,    39,    
+    103,   71,    39,    39,    39,    39,    39,    39,    71,    39,    
+    135,   231,   135,   135,   39,    327,   551,   103,   167,   551,   
+    89,    1434,  3226,  506,   474,   506,   506,   367,   1018,  1946,  
+    1402,  954,   1402,  314,   90,    1082,  218,   2266,  666,   1210,  
+    186,   570,   2042,  58,    5850,  154,   2010,  154,   794,   2266,  
+    378,   2266,  3738,  39,    39,    39,    39,    39,    39,    17351, 
+    34,    3074,  7692,  63,    63,    
+  };
+
+int sqlite3Fts5UnicodeCategory(int iCode) { 
+  int iRes = -1;
+  int iHi;
+  int iLo;
+  int ret;
+  u16 iKey;
+
+  if( iCode>=(1<<20) ){
+    return 0;
+  }
+  iLo = aFts5UnicodeBlock[(iCode>>16)];
+  iHi = aFts5UnicodeBlock[1+(iCode>>16)];
+  iKey = (iCode & 0xFFFF);
+  while( iHi>iLo ){
+    int iTest = (iHi + iLo) / 2;
+    assert( iTest>=iLo && iTest<iHi );
+    if( iKey>=aFts5UnicodeMap[iTest] ){
+      iRes = iTest;
+      iLo = iTest+1;
+    }else{
+      iHi = iTest;
+    }
+  }
+
+  if( iRes<0 ) return 0;
+  if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
+  ret = aFts5UnicodeData[iRes] & 0x1F;
+  if( ret!=30 ) return ret;
+  return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9;
+}
+
+void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
+  int i = 0;
+  int iTbl = 0;
+  while( i<128 ){
+    int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
+    int n = (aFts5UnicodeData[iTbl] >> 5) + i;
+    for(; i<128 && i<n; i++){
+      aAscii[i] = bToken;
+    }
+    iTbl++;
+  }
+}
@@ -431,6 +431,8 @@ static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
  i64 *pp = &pCsr->iInstPos;
  int *po = &pCsr->iInstOff;
  
+  assert( sqlite3Fts5IterEof(pIter)==0 );
+  assert( pCsr->bEof==0 );
  while( eDetail==FTS5_DETAIL_NONE
      || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) 
  ){
@@ -440,7 +442,7 @@ static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
    rc = sqlite3Fts5IterNextScan(pCsr->pIter);
    if( rc==SQLITE_OK ){
      rc = fts5VocabInstanceNewTerm(pCsr);
-      if( eDetail==FTS5_DETAIL_NONE ) break;
+      if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE ) break;
    }
    if( rc ){
      pCsr->bEof = 1;
@@ -755,10 +757,9 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
    /* xSavepoint    */ 0,
    /* xRelease      */ 0,
    /* xRollbackTo   */ 0,
+    /* xShadowName   */ 0
  };
  void *p = (void*)pGlobal;

  return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
 }
-
-
@@ -409,6 +409,7 @@ do_test 14.3 {
 do_execsql_test 15.0 {
  INSERT INTO t1(t1) VALUES('integrity-check');
 }
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 15.1 {
  UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1;
 }
@@ -591,6 +592,18 @@ do_execsql_test 22.1 {
  SELECT rowid FROM t9('a*')
 } {1}

+#-------------------------------------------------------------------------
+do_execsql_test 23.0 {
+  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
+  CREATE TABLE t11(x);
+}
+do_execsql_test 23.1 {
+  SELECT * FROM t11, t10 WHERE t11.x = t10.x AND t10.rowid IS NULL;
+}
+do_execsql_test 23.2 {
+  SELECT * FROM t11, t10 WHERE t10.rowid IS NULL;
+}
+
 }

 expand_all_sql db
@@ -0,0 +1,59 @@
+# 2016 Jan 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5cat
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize="unicode61 categories 'L*'");
+  INSERT INTO t1 VALUES ('Unlike1option2values3and4column5names');
+}
+
+do_execsql_test 1.1 {
+  SELECT rowid FROM t1('option');
+} {1}
+
+do_execsql_test 1.2 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x);
+  CREATE VIRTUAL TABLE t2t USING fts5vocab(t2, row);
+
+  CREATE VIRTUAL TABLE t3 USING fts5(
+    x, tokenize="unicode61 categories 'L* N* Co Mn'"
+  );
+  CREATE VIRTUAL TABLE t3t USING fts5vocab(t3, row);
+
+  CREATE VIRTUAL TABLE t4 USING fts5(
+    x, tokenize="unicode61 categories 'L* N* Co M*'"
+  );
+  CREATE VIRTUAL TABLE t4t USING fts5vocab(t4, row);
+
+  INSERT INTO t2 VALUES ('สนามกีฬา');
+  INSERT INTO t3 VALUES ('สนามกีฬา');
+  INSERT INTO t4 VALUES ('สนามกีฬา');
+}
+
+do_execsql_test 1.3 {
+  SELECT * FROM t2t
+} {สนามก 1 1 ฬา 1 1}
+
+do_execsql_test 1.4 {
+  SELECT * FROM t3t
+} {สนามกีฬา 1 1}
+
+do_execsql_test 1.5 {
+  SELECT * FROM t4t
+} {สนามกีฬา 1 1}
+
+
+finish_test
@@ -244,4 +244,3 @@ foreach {tn sql res} {
 }

 finish_test
-
@@ -41,6 +41,7 @@ db_save
 do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 set segid [lindex [fts5_level_segids t1] 0]

+sqlite3_db_config db DEFENSIVE 0
 do_test 1.3 {
  execsql {
    DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4);
@@ -50,6 +51,7 @@ do_test 1.3 {

 do_test 1.4 {
  db_restore_and_reopen
+  sqlite3_db_config db DEFENSIVE 0
  execsql {
    UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE
    rowid = fts5_rowid('segment', $segid, 4);
@@ -89,7 +91,7 @@ do_execsql_test 3.0 {
 do_execsql_test 3.1 {
  SELECT * FROM t3 WHERE t3 MATCH 'o'
 } {{one o} {three o} {five o}}
-
+sqlite3_db_config db DEFENSIVE 0
 do_catchsql_test 3.1 {
  DELETE FROM t3_content WHERE rowid = 3;
  SELECT * FROM t3 WHERE t3 MATCH 'o';
@@ -99,6 +99,7 @@ foreach {tno stmt} {
 set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] 
 set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
 set all [db eval {SELECT rowid FROM t1}]
+sqlite3_db_config db DEFENSIVE 0
 for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
  do_execsql_test 2.$i.1 {
    BEGIN;
@@ -248,6 +249,7 @@ foreach {tn hdr} {

 #--------------------------------------------------------------------
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE x5 USING fts5(tt);
  INSERT INTO x5 VALUES('a');
@@ -51,6 +51,7 @@ do_test 1.1 {
  set {} {}
 } {} 

+sqlite3_db_config db DEFENSIVE 0
 for {set i 0} {$i < $L} {incr i} {
  do_test 1.2.$i {
    catchsql {
@@ -86,6 +87,7 @@ do_execsql_test 2.2 {
 #
 reset_db
 do_test 3.0 { create_t1 } {}
+sqlite3_db_config db DEFENSIVE 0

 do_execsql_test 3.1 {
  SELECT count(*) FROM t1_data;
@@ -158,6 +160,7 @@ do_3_test 3.10
 # Test that segments that end unexpectedly are identified as corruption.
 #
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_test 4.0 {
  execsql { 
    CREATE VIRTUAL TABLE t1 USING fts5(x);
@@ -182,6 +185,7 @@ for {set i 1} {1} {incr i} {

  db close
  sqlite3 db test.db
+  sqlite3_db_config db DEFENSIVE 0

  db eval {
    BEGIN;
@@ -257,6 +261,7 @@ foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] {
 #------------------------------------------------------------------------
 #
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 6.1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a);
  INSERT INTO t1 VALUES('bbbbb ccccc');
@@ -273,6 +278,7 @@ do_catchsql_test 6.1.2 {

 #-------
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 6.2.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@@ -288,6 +294,7 @@ do_catchsql_test 6.2.2 {

 #-------
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 6.3.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a);
  INSERT INTO t1 VALUES('abc abcdef abcdefghi');
@@ -362,6 +369,7 @@ do_test 7.0 {
  }
 } {}

+sqlite3_db_config db DEFENSIVE 0
 do_test 7.1 {
  foreach i [db eval { SELECT rowid FROM t5_data WHERE rowid>100 }] {
    db eval BEGIN  
@@ -383,6 +391,7 @@ do_execsql_test 8.1 {
  INSERT INTO t1 VALUES('one', 'two');
 }

+sqlite3_db_config db DEFENSIVE 0
 do_test 9.1.1 {
  set    blob "12345678"    ;# cookie
  append blob "0105"        ;# 1 level, total of 5 segments
@@ -93,4 +93,3 @@ do_catchsql_test 3.3 {
  SELECT * FROM x2('^a');
 } {1 {fts5: phrase queries are not supported (detail!=full)}}
 finish_test
-
@@ -71,6 +71,7 @@ do_execsql_test 4.1 {
  INSERT INTO aa(aa) VALUES('integrity-check'); 
 }

+sqlite3_db_config db DEFENSIVE 0
 do_catchsql_test 4.2 { 
  BEGIN;
    UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;
@@ -29,38 +29,37 @@ do_execsql_test 1.0 {
 do_eqp_test 1.1 {
  SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
 } {
-  0 0 0 {SCAN TABLE t1} 
-  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
+  QUERY PLAN
+  |--SCAN TABLE t1
+  `--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
 }

 do_eqp_test 1.2 {
  SELECT * FROM t1, f1 WHERE f1 > t1.x
 } {
-  0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
-  0 1 0 {SCAN TABLE t1} 
+  QUERY PLAN
+  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
+  `--SCAN TABLE t1
 }

 do_eqp_test 1.3 {
  SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
 } {
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
-  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+  QUERY PLAN
+  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
+  `--USE TEMP B-TREE FOR ORDER BY
 }

 do_eqp_test 1.4 {
  SELECT * FROM f1 ORDER BY rank
 } {
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
-  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+  QUERY PLAN
+  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
+  `--USE TEMP B-TREE FOR ORDER BY
 }

 do_eqp_test 1.5 {
  SELECT * FROM f1 WHERE rank MATCH ?
-} {
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
-}
-
-
-
+} {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}

 finish_test
@@ -148,7 +148,18 @@ do_execsql_test 4.1 {
  VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank;
 } {{wrinkle in time} {Bill Smith}}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE ttt USING fts5(a);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO ttt SELECT 'word ' || i FROM s;
+}

-
+do_execsql_test 5.1 {
+  SELECT rowid FROM ttt('word') WHERE rowid BETWEEN 30 AND 40 ORDER BY rank;
+} {30 31 32 33 34 35 36 37 38 39 40}

 finish_test
@@ -39,6 +39,7 @@ do_execsql_test 1.4 {
  INSERT INTO f1(f1) VALUES('integrity-check');
 } {}

+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 1.5 {
  DELETE FROM f1_data;
 } {}
@@ -70,6 +70,7 @@ set res [db one {SELECT count(*) FROM x1_data}]
 do_execsql_test 2.3 {
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
 } $res
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 2.4 {
  UPDATE x1_data SET block = X'';
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
@@ -189,7 +189,7 @@ do_catchsql_test 6.2 {
 } {1 {error in tokenizer constructor}}
 do_catchsql_test 6.3 {
  CREATE VIRTUAL TABLE a3 USING fts5(
-    x, y, tokenize = 'unicode61 remove_diacritics 2'
+    x, y, tokenize = 'unicode61 remove_diacritics 3'
  );
 } {1 {error in tokenizer constructor}}
 do_catchsql_test 6.4 {
@@ -0,0 +1,65 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5umlaut
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      x, 
+      tokenize="unicode61 remove_diacritics 2"
+  );
+}
+
+foreach {tn q res1 res2} {
+  1 "Hà Nội"                  0 1
+  2 "Hà Noi"                  1 1
+  3 "Ha Noi"                  1 1
+  4 "Ha N\u1ed9i"             0 1
+  5 "Ha N\u006fi"             1 1
+  6 "Ha N\u006f\u0302i"       1 1
+  7 "Ha N\u006f\u0323\u0302i" 1 1
+} {
+  do_execsql_test 1.$tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t1($q)
+  } $res1
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t1('Ha Noi')
+  } $res1
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t2($q)
+  } $res2
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t2('Ha Noi')
+  } $res2
+}
+
+finish_test
+
@@ -41,7 +41,6 @@ foreach {tn t} {1 ascii 2 unicode61} {
 #-------------------------------------------------------------------------
 # Check that "unicode61" really is the default tokenizer.
 #
-
 do_execsql_test 2.0 "
  CREATE VIRTUAL TABLE t1 USING fts5(x);
  CREATE VIRTUAL TABLE t2 USING fts5(x, tokenize = unicode61);
@@ -56,5 +55,31 @@ do_execsql_test 2.1 "
  SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC';
 " {t1 t2}

+#-------------------------------------------------------------------------
+# Check that codepoints that require 4 bytes to store in utf-8 (those that
+# require 17 or more bits to store).
+#
+
+set A [db one {SELECT char(0x1F75E)}]    ;# Type So
+set B [db one {SELECT char(0x1F5FD)}]    ;# Type So
+set C [db one {SELECT char(0x2F802)}]    ;# Type Lo
+set D [db one {SELECT char(0x2F808)}]    ;# Type Lo
+
+do_execsql_test 3.0 "
+  CREATE VIRTUAL TABLE xyz USING fts5(x,
+    tokenize = \"unicode61 separators '$C' tokenchars '$A'\"
+  );
+  CREATE VIRTUAL TABLE xyz_v USING fts5vocab(xyz, row);
+
+  INSERT INTO xyz VALUES('$A$B$C$D');
+"
+
+do_execsql_test 3.1 {
+  SELECT * FROM xyz_v;
+} [list $A 1 1 $D 1 1]
+  
+
+
+

 finish_test
@@ -36,24 +36,26 @@ foreach x [an_load_unicodedata_text $UD] {
 }

 foreach {y} [rd_load_unicodedata_text $UD] {
-  foreach {code ascii} $y {}
+  foreach {code ascii f} $y {}
  if {$ascii==""} {
    set int 0
  } else {
    binary scan $ascii c int
  }
-  set aDiacritic($code) $int
+  set aDiacritic($code,$f) $int
+  if {$f==0} { set aDiacritic($code,1) $int }
 }

 proc tcl_fold {i {bRemoveDiacritic 0}} {
  global tl_lookup_table
  global aDiacritic
+  set f [expr $bRemoveDiacritic==2]

  if {[info exists tl_lookup_table($i)]} {
    set i $tl_lookup_table($i)
  }
-  if {$bRemoveDiacritic && [info exists aDiacritic($i)]} {
-    set i $aDiacritic($i)
+  if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} {
+    set i $aDiacritic($i,$f)
  }
  expr $i
 }
@@ -85,7 +87,7 @@ do_execsql_test 1.1 {
  SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
 } {0 {}}

-do_execsql_test 1.2 {
+do_execsql_test 1.2.1 {
  WITH ii(i) AS (
    SELECT -1
    UNION ALL
@@ -95,6 +97,16 @@ do_execsql_test 1.2 {
  WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
 } {0 {}}

+do_execsql_test 1.2.2 {
+  WITH ii(i) AS (
+    SELECT -1
+    UNION ALL
+    SELECT i+1 FROM ii WHERE i<100000
+  )
+  SELECT count(*), min(i) FROM ii 
+  WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int);
+} {0 {}}
+
 do_execsql_test 1.3 {
  WITH ii(i) AS (
    SELECT -1
@@ -0,0 +1,31 @@
+# 2018 July 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5unicode4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE sss USING fts5(a, prefix=3); 
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO sss VALUES('まりや');
+}
+
+finish_test
@@ -36,9 +36,10 @@ do_execsql_test 1.3 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'a';
 } {1}

+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 1.4 {
  UPDATE t1_config set v=5 WHERE k='version';
-} 
+}

 do_test 1.5 {
  db close
@@ -53,6 +54,7 @@ do_test 1.6 {
 } {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}

 do_test 1.7 {
+  sqlite3_db_config db DEFENSIVE 0
  execsql { DELETE FROM t1_config WHERE k='version' }
  db close
  sqlite3 db test.db
@@ -420,6 +420,7 @@ if {[detail_is_none]} { set resc [row_to_col $resr] }
 do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
 do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc

+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 8.2 {
  PRAGMA writable_schema = 1;
  UPDATE sqlite_master 
@@ -481,4 +482,3 @@ do_test 9.6 {


 finish_test
-
@@ -13,7 +13,7 @@
 #

 source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5vocab
+set testprefix fts5vocab2

 # If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
@@ -206,4 +206,3 @@ do_execsql_test 3.5 {
 }

 finish_test
-
@@ -39,8 +39,8 @@ SQLite. Documentation follows.
    To utilise "general" case mapping, the upper() or lower() scalar 
    functions are invoked with one argument:

-        upper('ABC') -> 'abc'
-        lower('abc') -> 'ABC'
+        upper('abc') -> 'ABC'
+        lower('ABC') -> 'abc'

    To access ICU "language specific" case mapping, upper() or lower()
    should be invoked with two arguments. The second argument is the name
@@ -274,6 +274,7 @@ static void doDataTest1(
  int rc = LSM_OK;
  Datasource *pData;
  TestDb *pDb;
+  int iToggle = 0;

  /* Start the test case, open a database and allocate the datasource. */
  pDb = testOpen(zSystem, 1, &rc);
@@ -287,8 +288,11 @@ static void doDataTest1(
    testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
    i += p->nVerify;

+    if( iToggle ) testBegin(pDb, 1, &rc);
    /* Check that the db content is correct. */
    testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
+    if( iToggle ) testCommit(pDb, 0, &rc);
+    iToggle = (iToggle+1)%2;

    if( bRecover ){
      testReopenRecover(&pDb, &rc);
@@ -617,8 +617,12 @@ static int test_lsm_fetch(

  if( pKey==0 ) return LSM_OK;

-  rc = lsm_csr_open(pDb->db, &csr);
-  if( rc!=LSM_OK ) return rc;
+  if( pDb->pCsr==0 ){
+    rc = lsm_csr_open(pDb->db, &csr);
+    if( rc!=LSM_OK ) return rc;
+  }else{
+    csr = pDb->pCsr;
+  }

  rc = lsm_csr_seek(csr, pKey, nKey, LSM_SEEK_EQ);
  if( rc==LSM_OK ){
@@ -638,7 +642,9 @@ static int test_lsm_fetch(
      *pnVal = -1;
    }
  }
-  lsm_csr_close(csr);
+  if( pDb->pCsr==0 ){
+    lsm_csr_close(csr);
+  }
  return rc;
 }

@@ -652,10 +658,28 @@ static int test_lsm_scan(
 ){
  LsmDb *pDb = (LsmDb *)pTestDb;
  lsm_cursor *csr;
+  lsm_cursor *csr2 = 0;
  int rc;

-  rc = lsm_csr_open(pDb->db, &csr);
-  if( rc!=LSM_OK ) return rc;
+  if( pDb->pCsr==0 ){
+    rc = lsm_csr_open(pDb->db, &csr);
+    if( rc!=LSM_OK ) return rc;
+  }else{
+    rc = LSM_OK;
+    csr = pDb->pCsr;
+  }
+
+  /* To enhance testing, if both pLast and pFirst are defined, seek the
+  ** cursor to the "end" boundary here. Then the next block seeks it to
+  ** the "start" ready for the scan. The point is to test that cursors
+  ** can be reused.  */
+  if( pLast && pFirst ){
+    if( bReverse ){
+      rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_LE);
+    }else{
+      rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_GE);
+    }
+  }

  if( bReverse ){
    if( pLast ){
@@ -696,7 +720,9 @@ static int test_lsm_scan(
    }
  }

-  lsm_csr_close(csr);
+  if( pDb->pCsr==0 ){
+    lsm_csr_close(csr);
+  }
  return rc;
 }

@@ -762,6 +788,7 @@ static void xWorkHook(lsm_db *db, void *pArg){
 #define TEST_MT_MIN_CKPT -4
 #define TEST_MT_MAX_CKPT -5

+
 int test_lsm_config_str(
  LsmDb *pLsm,
  lsm_db *db, 
@@ -1922,6 +1922,7 @@ static int seekInLevel(
  ** is not a composite level and there is no split-key). Search the 
  ** left-hand-side of the level in this case.  */
  if( res<0 ){
+    int i;
    int iPtr = 0;
    if( nRhs==0 ) iPtr = (int)*piPgno;

@@ -1931,12 +1932,16 @@ static int seekInLevel(
    if( rc==LSM_OK && nRhs>0 && eSeek==LSM_SEEK_GE && aPtr[0].pPg==0 ){
      res = 0;
    }
+    for(i=1; i<=nRhs; i++){
+      segmentPtrReset(&aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD);
+    }
  }
  
  if( res>=0 ){
    int bHit = 0;                 /* True if at least one rhs is not EOF */
    int iPtr = (int)*piPgno;
    int i;
+    segmentPtrReset(&aPtr[0], LSM_SEGMENTPTR_FREE_THRESHOLD);
    for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){
      SegmentPtr *pPtr = &aPtr[i];
      iOut = 0;
@@ -2868,7 +2873,7 @@ static int multiCursorEnd(MultiCursor *pCsr, int bLast){
  int rc = LSM_OK;
  int i;

-  pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK);
+  pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ);
  pCsr->flags |= (bLast ? CURSOR_PREV_OK : CURSOR_NEXT_OK);
  pCsr->iFree = 0;

@@ -1473,7 +1473,8 @@ static sqlite3_module amatchModule = {
  0,                      /* xRename */
  0,                      /* xSavepoint */
  0,                      /* xRelease */
-  0                       /* xRollbackTo */
+  0,                      /* xRollbackTo */
+  0                       /* xShadowName */
 };

 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -411,6 +411,7 @@ int sqlite3BinfoRegister(sqlite3 *db){
    0,                           /* xSavepoint */
    0,                           /* xRelease */
    0,                           /* xRollbackTo */
+    0                            /* xShadowName */
  };
  return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0);
 }
@@ -826,17 +826,12 @@ static int closureBestIndex(
  int iPlan = 0;
  int i;
  int idx = 1;
-  int seenMatch = 0;
  const struct sqlite3_index_constraint *pConstraint;
  closure_vtab *pVtab = (closure_vtab*)pTab;
  double rCost = 10000000.0;

  pConstraint = pIdxInfo->aConstraint;
  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->iColumn==CLOSURE_COL_ROOT
-     && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
-      seenMatch = 1;
-    }
    if( pConstraint->usable==0 ) continue;
    if( (iPlan & 1)==0 
     && pConstraint->iColumn==CLOSURE_COL_ROOT
@@ -893,6 +888,18 @@ static int closureBestIndex(
    ** or else the result is an empty set. */
    iPlan = 0;
  }
+  if( (iPlan&1)==0 ){
+    /* If there is no usable "root=?" term, then set the index-type to 0.
+    ** Also clear any argvIndex variables already set. This is necessary
+    ** to prevent the core from throwing an "xBestIndex malfunction error"
+    ** error (because the argvIndex values are not contiguously assigned
+    ** starting from 1).  */
+    rCost *= 1e30;
+    for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
+      pIdxInfo->aConstraintUsage[i].argvIndex = 0;
+    }
+    iPlan = 0;
+  }
  pIdxInfo->idxNum = iPlan;
  if( pIdxInfo->nOrderBy==1
   && pIdxInfo->aOrderBy[0].iColumn==CLOSURE_COL_ID
@@ -900,7 +907,6 @@ static int closureBestIndex(
  ){
    pIdxInfo->orderByConsumed = 1;
  }
-  if( seenMatch && (iPlan&1)==0 ) rCost *= 1e30;
  pIdxInfo->estimatedCost = rCost;
   
  return SQLITE_OK;
@@ -932,7 +938,8 @@ static sqlite3_module closureModule = {
  0,                      /* xRename */
  0,                      /* xSavepoint */
  0,                      /* xRelease */
-  0                       /* xRollbackTo */
+  0,                      /* xRollbackTo */
+  0                       /* xShadowName */
 };

 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -62,6 +62,7 @@ struct completion_cursor {
  char *zPrefix;             /* The prefix for the word we want to complete */
  char *zLine;               /* The whole that we want to complete */
  const char *zCurrentRow;   /* Current output row */
+  int szRow;                 /* Length of the zCurrentRow string */
  sqlite3_stmt *pStmt;       /* Current statement */
  sqlite3_int64 iRowid;      /* The rowid */
  int ePhase;                /* Current phase */
@@ -78,7 +79,7 @@ struct completion_cursor {
 #define COMPLETION_INDEXES       5
 #define COMPLETION_TRIGGERS      6
 #define COMPLETION_DATABASES     7
-#define COMPLETION_TABLES        8
+#define COMPLETION_TABLES        8    /* Also VIEWs and TRIGGERs */
 #define COMPLETION_COLUMNS       9
 #define COMPLETION_MODULES       10
 #define COMPLETION_EOF           11
@@ -174,32 +175,6 @@ static int completionClose(sqlite3_vtab_cursor *cur){
  return SQLITE_OK;
 }

-/*
-** All SQL keywords understood by SQLite
-*/
-static const char *completionKwrds[] = {
-  "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-  "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-  "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-  "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-  "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-  "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-  "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-  "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-  "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-  "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-  "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-  "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-  "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-  "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-  "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-  "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-  "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-  "WITH", "WITHOUT",
-};
-#define completionKwCount \
-   (int)(sizeof(completionKwrds)/sizeof(completionKwrds[0]))
-
 /*
 ** Advance a completion_cursor to its next row of output.
 **
@@ -222,11 +197,11 @@ static int completionNext(sqlite3_vtab_cursor *cur){
  while( pCur->ePhase!=COMPLETION_EOF ){
    switch( pCur->ePhase ){
      case COMPLETION_KEYWORDS: {
-        if( pCur->j >= completionKwCount ){
+        if( pCur->j >= sqlite3_keyword_count() ){
          pCur->zCurrentRow = 0;
          pCur->ePhase = COMPLETION_DATABASES;
        }else{
-          pCur->zCurrentRow = completionKwrds[pCur->j++];
+          sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow);
        }
        iCol = -1;
        break;
@@ -250,8 +225,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){
            const char *zDb = (const char*)sqlite3_column_text(pS2, 1);
            zSql = sqlite3_mprintf(
               "%z%s"
-               "SELECT name FROM \"%w\".sqlite_master"
-               " WHERE type='table'",
+               "SELECT name FROM \"%w\".sqlite_master",
               zSql, zSep, zDb
            );
            if( zSql==0 ) return SQLITE_NOMEM;
@@ -299,6 +273,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){
      if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){
        /* Extract the next row of content */
        pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol);
+        pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol);
      }else{
        /* When all rows are finished, advance to the next phase */
        sqlite3_finalize(pCur->pStmt);
@@ -308,7 +283,9 @@ static int completionNext(sqlite3_vtab_cursor *cur){
      }
    }
    if( pCur->nPrefix==0 ) break;
-    if( sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 ){
+    if( pCur->nPrefix<=pCur->szRow
+     && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0
+    ){
      break;
    }
  }
@@ -328,7 +305,7 @@ static int completionColumn(
  completion_cursor *pCur = (completion_cursor*)cur;
  switch( i ){
    case COMPLETION_COLUMN_CANDIDATE: {
-      sqlite3_result_text(ctx, pCur->zCurrentRow, -1, SQLITE_TRANSIENT);
+      sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT);
      break;
    }
    case COMPLETION_COLUMN_PREFIX: {
@@ -388,7 +365,7 @@ static int completionFilter(
      pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg]));
      if( pCur->zPrefix==0 ) return SQLITE_NOMEM;
    }
-    iArg++;
+    iArg = 1;
  }
  if( idxNum & 2 ){
    pCur->nLine = sqlite3_value_bytes(argv[iArg]);
@@ -396,7 +373,6 @@ static int completionFilter(
      pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg]));
      if( pCur->zLine==0 ) return SQLITE_NOMEM;
    }
-    iArg++;
  }
  if( pCur->zLine!=0 && pCur->zPrefix==0 ){
    int i = pCur->nLine;
@@ -492,7 +468,8 @@ static sqlite3_module completionModule = {
  0,                         /* xRename */
  0,                         /* xSavepoint */
  0,                         /* xRelease */
-  0                          /* xRollbackTo */
+  0,                         /* xRollbackTo */
+  0                          /* xShadowName */
 };

 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -19,9 +19,9 @@
 **    CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
 **    SELECT * FROM csv;
 **
-** The columns are named "c1", "c2", "c3", ... by default.  But the
-** application can define its own CREATE TABLE statement as an additional
-** parameter.  For example:
+** The columns are named "c1", "c2", "c3", ... by default.  Or the
+** application can define its own CREATE TABLE statement using the
+** schema= parameter, like this:
 **
 **    CREATE VIRTUAL TABLE temp.csv2 USING csv(
 **       filename = "../http.log",
@@ -32,9 +32,9 @@
 ** the data= parameter.
 **
 ** If the columns=N parameter is supplied, then the CSV file is assumed to have
-** N columns.  If the columns parameter is omitted, the CSV file is opened
-** as soon as the virtual table is constructed and the first row of the CSV
-** is read in order to count the tables.
+** N columns.  If both the columns= and schema= parameters are omitted, then
+** the number and names of the columns is determined by the first line of
+** the CSV input.
 **
 ** Some extra debugging features (used for testing virtual tables) are available
 ** if this module is compiled with -DSQLITE_TEST.
@@ -132,6 +132,7 @@ static int csv_reader_open(
    }
    p->in = fopen(zFilename, "rb");
    if( p->in==0 ){
+      sqlite3_free(p->zIn);
      csv_reader_reset(p);
      csv_errmsg(p, "cannot open '%s' for reading", zFilename);
      return 1;
@@ -204,7 +205,8 @@ static int csv_append(CsvReader *p, char c){
 **   +  Store the character that terminates the field in p->cTerm.  Store
 **      EOF on end-of-file.
 **
-** Return "" at EOF.  Return 0 on an OOM error.
+** Return 0 at EOF or on OOM.  On EOF, the p->cTerm character will have
+** been set to EOF.
 */
 static char *csv_read_one_field(CsvReader *p){
  int c;
@@ -212,7 +214,7 @@ static char *csv_read_one_field(CsvReader *p){
  c = csv_getc(p);
  if( c==EOF ){
    p->cTerm = EOF;
-    return "";
+    return 0;
  }
  if( c=='"' ){
    int pc, ppc;
@@ -434,6 +436,34 @@ static int csv_boolean(const char *z){
  return -1;
 }

+/* Check to see if the string is of the form:  "TAG = BOOLEAN" or just "TAG".
+** If it is, set *pValue to be the value of the boolean ("true" if there is
+** not "= BOOLEAN" component) and return non-zero.  If the input string
+** does not begin with TAG, return zero.
+*/
+static int csv_boolean_parameter(
+  const char *zTag,       /* Tag we are looking for */
+  int nTag,               /* Size of the tag in bytes */
+  const char *z,          /* Input parameter */
+  int *pValue             /* Write boolean value here */
+){
+  int b;
+  z = csv_skip_whitespace(z);
+  if( strncmp(zTag, z, nTag)!=0 ) return 0;
+  z = csv_skip_whitespace(z + nTag);
+  if( z[0]==0 ){
+    *pValue = 1;
+    return 1;
+  }
+  if( z[0]!='=' ) return 0;
+  z = csv_skip_whitespace(z+1);
+  b = csv_boolean(z);
+  if( b>=0 ){
+    *pValue = b;
+    return 1;
+  }
+  return 0;
+}

 /*
 ** Parameters:
@@ -467,6 +497,7 @@ static int csvtabConnect(
 #ifdef SQLITE_TEST
  int tstFlags = 0;          /* Value for testflags=N parameter */
 #endif
+  int b;                     /* Value of a boolean parameter */
  int nCol = -99;            /* Value of the columns= parameter */
  CsvReader sRdr;            /* A CSV file reader used to store an error
                             ** message and/or to count the number of columns */
@@ -491,21 +522,12 @@ static int csvtabConnect(
    if( j<sizeof(azParam)/sizeof(azParam[0]) ){
      if( sRdr.zErr[0] ) goto csvtab_connect_error;
    }else
-    if( (zValue = csv_parameter("header",6,z))!=0 ){
-      int x;
+    if( csv_boolean_parameter("header",6,z,&b) ){
      if( bHeader>=0 ){
        csv_errmsg(&sRdr, "more than one 'header' parameter");
        goto csvtab_connect_error;
      }
-      x = csv_boolean(zValue);
-      if( x==1 ){
-        bHeader = 1;
-      }else if( x==0 ){
-        bHeader = 0;
-      }else{
-        csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue);
-        goto csvtab_connect_error;
-      }
+      bHeader = b;
    }else
 #ifdef SQLITE_TEST
    if( (zValue = csv_parameter("testflags",9,z))!=0 ){
@@ -519,54 +541,94 @@ static int csvtabConnect(
      }
      nCol = atoi(zValue);
      if( nCol<=0 ){
-        csv_errmsg(&sRdr, "must have at least one column");
+        csv_errmsg(&sRdr, "column= value must be positive");
        goto csvtab_connect_error;
      }
    }else
    {
-      csv_errmsg(&sRdr, "unrecognized parameter '%s'", z);
+      csv_errmsg(&sRdr, "bad parameter: '%s'", z);
      goto csvtab_connect_error;
    }
  }
  if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
-    csv_errmsg(&sRdr, "must either filename= or data= but not both");
+    csv_errmsg(&sRdr, "must specify either filename= or data= but not both");
    goto csvtab_connect_error;
  }
-  if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){
+
+  if( (nCol<=0 || bHeader==1)
+   && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA)
+  ){
    goto csvtab_connect_error;
  }
  pNew = sqlite3_malloc( sizeof(*pNew) );
  *ppVtab = (sqlite3_vtab*)pNew;
  if( pNew==0 ) goto csvtab_connect_oom;
  memset(pNew, 0, sizeof(*pNew));
-  if( nCol>0 ){
+  if( CSV_SCHEMA==0 ){
+    sqlite3_str *pStr = sqlite3_str_new(0);
+    char *zSep = "";
+    int iCol = 0;
+    sqlite3_str_appendf(pStr, "CREATE TABLE x(");
+    if( nCol<0 && bHeader<1 ){
+      nCol = 0;
+      do{
+        csv_read_one_field(&sRdr);
+        nCol++;
+      }while( sRdr.cTerm==',' );
+    }
+    if( nCol>0 && bHeader<1 ){
+      for(iCol=0; iCol<nCol; iCol++){
+        sqlite3_str_appendf(pStr, "%sc%d TEXT", zSep, iCol);
+        zSep = ",";
+      }
+    }else{
+      do{
+        char *z = csv_read_one_field(&sRdr);
+        if( (nCol>0 && iCol<nCol) || (nCol<0 && bHeader) ){
+          sqlite3_str_appendf(pStr,"%s\"%w\" TEXT", zSep, z);
+          zSep = ",";
+          iCol++;
+        }
+      }while( sRdr.cTerm==',' );
+      if( nCol<0 ){
+        nCol = iCol;
+      }else{
+        while( iCol<nCol ){
+          sqlite3_str_appendf(pStr,"%sc%d TEXT", zSep, ++iCol);
+          zSep = ",";
+        }
+      }
+    }
    pNew->nCol = nCol;
-  }else{
+    sqlite3_str_appendf(pStr, ")");
+    CSV_SCHEMA = sqlite3_str_finish(pStr);
+    if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
+  }else if( nCol<0 ){
    do{
-      const char *z = csv_read_one_field(&sRdr);
-      if( z==0 ) goto csvtab_connect_oom;
+      csv_read_one_field(&sRdr);
      pNew->nCol++;
    }while( sRdr.cTerm==',' );
+  }else{
+    pNew->nCol = nCol;
  }
  pNew->zFilename = CSV_FILENAME;  CSV_FILENAME = 0;
  pNew->zData = CSV_DATA;          CSV_DATA = 0;
 #ifdef SQLITE_TEST
  pNew->tstFlags = tstFlags;
 #endif
-  pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0;
-  csv_reader_reset(&sRdr);
-  if( CSV_SCHEMA==0 ){
-    char *zSep = "";
-    CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x(");
-    if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
-    for(i=0; i<pNew->nCol; i++){
-      CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i);
-      zSep = ",";
-    }
-    CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA);
+  if( bHeader!=1 ){
+    pNew->iStart = 0;
+  }else if( pNew->zData ){
+    pNew->iStart = (int)sRdr.iIn;
+  }else{
+    pNew->iStart = ftell(sRdr.in);
  }
+  csv_reader_reset(&sRdr);
  rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
-  if( rc ) goto csvtab_connect_error;
+  if( rc ){
+    csv_errmsg(&sRdr, "bad schema: '%s' - %s", CSV_SCHEMA, sqlite3_errmsg(db));
+    goto csvtab_connect_error;
+  }
  for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
    sqlite3_free(azPValue[i]);
  }
@@ -662,7 +724,6 @@ static int csvtabNext(sqlite3_vtab_cursor *cur){
  do{
    z = csv_read_one_field(&pCur->rdr);
    if( z==0 ){
-      csv_xfer_error(pTab, &pCur->rdr);
      break;
    }
    if( i<pTab->nCol ){
@@ -141,45 +141,12 @@ static void appendText(DText *p, char const *zAppend, char quote){
 ** Return '"' if quoting is required.  Return 0 if no quoting is required.
 */
 static char quoteChar(const char *zName){
-  /* All SQLite keywords, in alphabetical order */
-  static const char *azKeywords[] = {
-    "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-    "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-    "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-    "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-    "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-    "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-    "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-    "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-    "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-    "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-    "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-    "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-    "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-    "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-    "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-    "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-    "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-    "WITH", "WITHOUT",
-  };
-  int i, lwr, upr, mid, c;
+  int i;
  if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
  for(i=0; zName[i]; i++){
    if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
  }
-  lwr = 0;
-  upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
-  while( lwr<=upr ){
-    mid = (lwr+upr)/2;
-    c = sqlite3_stricmp(azKeywords[mid], zName);
-    if( c==0 ) return '"';
-    if( c<0 ){
-      lwr = mid+1;
-    }else{
-      upr = mid-1;
-    }
-  }
-  return 0;
+  return sqlite3_keyword_check(zName, i) ? '"' : 0;
 }


@@ -518,7 +485,15 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
            }
            case SQLITE_FLOAT: {
              double r = sqlite3_column_double(pStmt,i);
-              output_formatted(p, "%!.20g", r);
+              sqlite3_uint64 ur;
+              memcpy(&ur,&r,sizeof(r));
+              if( ur==0x7ff0000000000000LL ){
+                p->xCallback("1e999", p->pArg);
+              }else if( ur==0xfff0000000000000LL ){
+                p->xCallback("-1e999", p->pArg);
+              }else{
+                output_formatted(p, "%!.20g", r);
+              }
              break;
            }
            case SQLITE_NULL: {
@@ -34,6 +34,7 @@ struct EvalResult {
 static int callback(void *pCtx, int argc, char **argv, char **colnames){
  struct EvalResult *p = (struct EvalResult*)pCtx;
  int i; 
+  if( argv==0 ) return 0;
  for(i=0; i<argc; i++){
    const char *z = argv[i] ? argv[i] : "";
    size_t sz = strlen(z);
@@ -0,0 +1,322 @@
+/*
+** 2018-09-16
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file demonstrates an eponymous virtual table that returns the
+** EXPLAIN output from an SQL statement.
+**
+** Usage example:
+**
+**     .load ./explain
+**     SELECT p2 FROM explain('SELECT * FROM sqlite_master')
+**      WHERE opcode='OpenRead';
+**
+** This module was originally written to help simplify SQLite testing,
+** by providing an easier means of verifying certain patterns in the
+** generated bytecode.
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* explain_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a explain virtual table
+*/
+typedef struct explain_vtab explain_vtab;
+struct explain_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  sqlite3 *db;        /* Database connection for this explain vtab */
+};
+
+/* explain_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result from an EXPLAIN operation.
+*/
+typedef struct explain_cursor explain_cursor;
+struct explain_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  sqlite3 *db;               /* Database connection for this cursor */
+  char *zSql;                /* Value for the EXPLN_COLUMN_SQL column */
+  sqlite3_stmt *pExplain;    /* Statement being explained */
+  int rc;                    /* Result of last sqlite3_step() on pExplain */
+};
+
+/*
+** The explainConnect() method is invoked to create a new
+** explain_vtab that describes the explain virtual table.
+**
+** Think of this routine as the constructor for explain_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the explain_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against explain will look like.
+*/
+static int explainConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  explain_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define EXPLN_COLUMN_ADDR     0   /* Instruction address */
+#define EXPLN_COLUMN_OPCODE   1   /* Opcode */
+#define EXPLN_COLUMN_P1       2   /* Operand 1 */
+#define EXPLN_COLUMN_P2       3   /* Operand 2 */
+#define EXPLN_COLUMN_P3       4   /* Operand 3 */
+#define EXPLN_COLUMN_P4       5   /* Operand 4 */
+#define EXPLN_COLUMN_P5       6   /* Operand 5 */
+#define EXPLN_COLUMN_COMMENT  7   /* Comment */
+#define EXPLN_COLUMN_SQL      8   /* SQL that is being explained */
+
+
+  rc = sqlite3_declare_vtab(db,
+     "CREATE TABLE x(addr,opcode,p1,p2,p3,p4,p5,comment,sql HIDDEN)");
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->db = db;
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for explain_cursor objects.
+*/
+static int explainDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new explain_cursor object.
+*/
+static int explainOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  explain_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->db = ((explain_vtab*)p)->db;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a explain_cursor.
+*/
+static int explainClose(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  sqlite3_finalize(pCur->pExplain);
+  sqlite3_free(pCur->zSql);
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a explain_cursor to its next row of output.
+*/
+static int explainNext(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  pCur->rc = sqlite3_step(pCur->pExplain);
+  if( pCur->rc!=SQLITE_DONE && pCur->rc!=SQLITE_ROW ) return pCur->rc;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the explain_cursor
+** is currently pointing.
+*/
+static int explainColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  if( i==EXPLN_COLUMN_SQL ){
+    sqlite3_result_text(ctx, pCur->zSql, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_value(ctx, sqlite3_column_value(pCur->pExplain, i));
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int explainRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  *pRowid = sqlite3_column_int64(pCur->pExplain, 0);
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int explainEof(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  return pCur->rc!=SQLITE_ROW;
+}
+
+/*
+** This method is called to "rewind" the explain_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to explainColumn() or explainRowid() or 
+** explainEof().
+**
+** The argv[0] is the SQL statement that is to be explained.
+*/
+static int explainFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  explain_cursor *pCur = (explain_cursor *)pVtabCursor;
+  char *zSql = 0;
+  int rc;
+  sqlite3_finalize(pCur->pExplain);
+  pCur->pExplain = 0;
+  if( sqlite3_value_type(argv[0])!=SQLITE_TEXT ){
+    pCur->rc = SQLITE_DONE;
+    return SQLITE_OK;
+  }
+  sqlite3_free(pCur->zSql);
+  pCur->zSql = sqlite3_mprintf("%s", sqlite3_value_text(argv[0]));
+  if( pCur->zSql ){
+    zSql = sqlite3_mprintf("EXPLAIN %s", pCur->zSql);
+  }
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pExplain, 0);
+    sqlite3_free(zSql);
+  }
+  if( rc ){
+    sqlite3_finalize(pCur->pExplain);
+    pCur->pExplain = 0;
+    sqlite3_free(pCur->zSql);
+    pCur->zSql = 0;
+  }else{
+    pCur->rc = sqlite3_step(pCur->pExplain);
+    rc = (pCur->rc==SQLITE_DONE || pCur->rc==SQLITE_ROW) ? SQLITE_OK : pCur->rc;
+  }
+  return rc;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the explain virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int explainBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;                  /* Loop counter */
+  int idx = -1;           /* Index of a usable == constraint against SQL */
+  int unusable = 0;       /* True if there are unusable constraints on SQL */
+
+  pIdxInfo->estimatedRows = 500;
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+    if( p->iColumn!=EXPLN_COLUMN_SQL ) continue;
+    if( !p->usable ){
+      unusable = 1;
+    }else if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      idx = i;
+    }
+  }
+  if( idx>=0 ){
+    /* There exists a usable == constraint against the SQL column */
+    pIdxInfo->estimatedCost = 10.0;
+    pIdxInfo->idxNum = 1;
+    pIdxInfo->aConstraintUsage[idx].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[idx].omit = 1;
+  }else if( unusable ){
+    /* There are unusable constraints against the SQL column.  Do not allow
+    ** this plan to continue forward. */
+    return SQLITE_CONSTRAINT;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** explain virtual table.
+*/
+static sqlite3_module explainModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  explainConnect,            /* xConnect */
+  explainBestIndex,          /* xBestIndex */
+  explainDisconnect,         /* xDisconnect */
+  0,                         /* xDestroy */
+  explainOpen,               /* xOpen - open a cursor */
+  explainClose,              /* xClose - close a cursor */
+  explainFilter,             /* xFilter - configure scan constraints */
+  explainNext,               /* xNext - advance a cursor */
+  explainEof,                /* xEof - check for end of scan */
+  explainColumn,             /* xColumn - read data */
+  explainRowid,              /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+int sqlite3ExplainVtabInit(sqlite3 *db){
+  int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3_create_module(db, "explain", &explainModule, 0);
+#endif
+  return rc;
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_explain_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3ExplainVtabInit(db);
+#endif
+  return rc;
+}
@@ -93,6 +93,9 @@ SQLITE_EXTENSION_INIT1
 #  include <direct.h>
 #  include "test_windirent.h"
 #  define dirent DIRENT
+#  ifndef chmod
+#    define chmod _chmod
+#  endif
 #  ifndef stat
 #    define stat _stat
 #  endif
@@ -103,7 +106,18 @@ SQLITE_EXTENSION_INIT1
 #include <errno.h>


+/*
+** Structure of the fsdir() table-valued function
+*/
+                 /*    0    1    2     3    4           5             */
 #define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
+#define FSDIR_COLUMN_NAME     0     /* Name of the file */
+#define FSDIR_COLUMN_MODE     1     /* Access mode */
+#define FSDIR_COLUMN_MTIME    2     /* Last modification time */
+#define FSDIR_COLUMN_DATA     3     /* File content */
+#define FSDIR_COLUMN_PATH     4     /* Path to top of search */
+#define FSDIR_COLUMN_DIR      5     /* Path is relative to this directory */
+

 /*
 ** Set the result stored by context ctx to a blob containing the 
@@ -159,6 +173,97 @@ static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
  va_end(ap);
 }

+#if defined(_WIN32)
+/*
+** This function is designed to convert a Win32 FILETIME structure into the
+** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC).
+*/
+static sqlite3_uint64 fileTimeToUnixTime(
+  LPFILETIME pFileTime
+){
+  SYSTEMTIME epochSystemTime;
+  ULARGE_INTEGER epochIntervals;
+  FILETIME epochFileTime;
+  ULARGE_INTEGER fileIntervals;
+
+  memset(&epochSystemTime, 0, sizeof(SYSTEMTIME));
+  epochSystemTime.wYear = 1970;
+  epochSystemTime.wMonth = 1;
+  epochSystemTime.wDay = 1;
+  SystemTimeToFileTime(&epochSystemTime, &epochFileTime);
+  epochIntervals.LowPart = epochFileTime.dwLowDateTime;
+  epochIntervals.HighPart = epochFileTime.dwHighDateTime;
+
+  fileIntervals.LowPart = pFileTime->dwLowDateTime;
+  fileIntervals.HighPart = pFileTime->dwHighDateTime;
+
+  return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000;
+}
+
+/*
+** This function attempts to normalize the time values found in the stat()
+** buffer to UTC.  This is necessary on Win32, where the runtime library
+** appears to return these values as local times.
+*/
+static void statTimesToUtc(
+  const char *zPath,
+  struct stat *pStatBuf
+){
+  HANDLE hFindFile;
+  WIN32_FIND_DATAW fd;
+  LPWSTR zUnicodeName;
+  extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
+  zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath);
+  if( zUnicodeName ){
+    memset(&fd, 0, sizeof(WIN32_FIND_DATAW));
+    hFindFile = FindFirstFileW(zUnicodeName, &fd);
+    if( hFindFile!=NULL ){
+      pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime);
+      pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime);
+      pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime);
+      FindClose(hFindFile);
+    }
+    sqlite3_free(zUnicodeName);
+  }
+}
+#endif
+
+/*
+** This function is used in place of stat().  On Windows, special handling
+** is required in order for the included time to be returned as UTC.  On all
+** other systems, this function simply calls stat().
+*/
+static int fileStat(
+  const char *zPath,
+  struct stat *pStatBuf
+){
+#if defined(_WIN32)
+  int rc = stat(zPath, pStatBuf);
+  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
+  return rc;
+#else
+  return stat(zPath, pStatBuf);
+#endif
+}
+
+/*
+** This function is used in place of lstat().  On Windows, special handling
+** is required in order for the included time to be returned as UTC.  On all
+** other systems, this function simply calls lstat().
+*/
+static int fileLinkStat(
+  const char *zPath,
+  struct stat *pStatBuf
+){
+#if defined(_WIN32)
+  int rc = lstat(zPath, pStatBuf);
+  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
+  return rc;
+#else
+  return lstat(zPath, pStatBuf);
+#endif
+}
+
 /*
 ** Argument zFile is the name of a file that will be created and/or written
 ** by SQL function writefile(). This function ensures that the directory
@@ -190,7 +295,7 @@ static int makeDirectory(
      if( i==nCopy ) break;
      zCopy[i] = '\0';

-      rc2 = stat(zCopy, &sStat);
+      rc2 = fileStat(zCopy, &sStat);
      if( rc2!=0 ){
        if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR;
      }else{
@@ -232,7 +337,7 @@ static int writeFile(
        ** to do so using chmod(), it is not an error.  */
        struct stat sStat;
        if( errno!=EEXIST
-         || 0!=stat(zFile, &sStat)
+         || 0!=fileStat(zFile, &sStat)
         || !S_ISDIR(sStat.st_mode)
         || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile, mode&0777))
        ){
@@ -279,6 +384,9 @@ static int writeFile(
    lastWrite.dwLowDateTime = (DWORD)intervals;
    lastWrite.dwHighDateTime = intervals >> 32;
    zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile);
+    if( zUnicodeName==0 ){
+      return 1;
+    }
    hFile = CreateFileW(
      zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
      FILE_FLAG_BACKUP_SEMANTICS, NULL
@@ -291,7 +399,7 @@ static int writeFile(
    }else{
      return 1;
    }
-#elif defined(AT_FDCWD) && 0 /* utimensat() is not univerally available */
+#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */
    /* Recent unix */
    struct timespec times[2];
    times[0].tv_nsec = times[1].tv_nsec = 0;
@@ -568,7 +676,7 @@ static int fsdirNext(sqlite3_vtab_cursor *cur){
      sqlite3_free(pCur->zPath);
      pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
      if( pCur->zPath==0 ) return SQLITE_NOMEM;
-      if( lstat(pCur->zPath, &pCur->sStat) ){
+      if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
        fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
        return SQLITE_ERROR;
      }
@@ -598,20 +706,20 @@ static int fsdirColumn(
 ){
  fsdir_cursor *pCur = (fsdir_cursor*)cur;
  switch( i ){
-    case 0: { /* name */
+    case FSDIR_COLUMN_NAME: {
      sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT);
      break;
    }

-    case 1: /* mode */
+    case FSDIR_COLUMN_MODE:
      sqlite3_result_int64(ctx, pCur->sStat.st_mode);
      break;

-    case 2: /* mtime */
+    case FSDIR_COLUMN_MTIME:
      sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
      break;

-    case 3: { /* data */
+    case FSDIR_COLUMN_DATA: {
      mode_t m = pCur->sStat.st_mode;
      if( S_ISDIR(m) ){
        sqlite3_result_null(ctx);
@@ -641,6 +749,12 @@ static int fsdirColumn(
        readFileContents(ctx, pCur->zPath);
      }
    }
+    case FSDIR_COLUMN_PATH:
+    default: {
+      /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters.
+      ** always return their values as NULL */
+      break;
+    }
  }
  return SQLITE_OK;
 }
@@ -667,6 +781,9 @@ static int fsdirEof(sqlite3_vtab_cursor *cur){

 /*
 ** xFilter callback.
+**
+** idxNum==1   PATH parameter only
+** idxNum==2   Both PATH and DIR supplied
 */
 static int fsdirFilter(
  sqlite3_vtab_cursor *cur, 
@@ -702,7 +819,7 @@ static int fsdirFilter(
  if( pCur->zPath==0 ){
    return SQLITE_NOMEM;
  }
-  if( lstat(pCur->zPath, &pCur->sStat) ){
+  if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
    fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
    return SQLITE_ERROR;
  }
@@ -719,40 +836,63 @@ static int fsdirFilter(
 ** In this implementation idxNum is used to represent the
 ** query plan.  idxStr is unused.
 **
-** The query plan is represented by bits in idxNum:
+** The query plan is represented by values of idxNum:
 **
-**  (1)  start = $value  -- constraint exists
-**  (2)  stop = $value   -- constraint exists
-**  (4)  step = $value   -- constraint exists
-**  (8)  output in descending order
+**  (1)  The path value is supplied by argv[0]
+**  (2)  Path is in argv[0] and dir is in argv[1]
 */
 static int fsdirBestIndex(
  sqlite3_vtab *tab,
  sqlite3_index_info *pIdxInfo
 ){
  int i;                 /* Loop over constraints */
-  int idx4 = -1;
-  int idx5 = -1;
+  int idxPath = -1;      /* Index in pIdxInfo->aConstraint of PATH= */
+  int idxDir = -1;       /* Index in pIdxInfo->aConstraint of DIR= */
+  int seenPath = 0;      /* True if an unusable PATH= constraint is seen */
+  int seenDir = 0;       /* True if an unusable DIR= constraint is seen */
  const struct sqlite3_index_constraint *pConstraint;

  (void)tab;
  pConstraint = pIdxInfo->aConstraint;
  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->usable==0 ) continue;
    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    if( pConstraint->iColumn==4 ) idx4 = i;
-    if( pConstraint->iColumn==5 ) idx5 = i;
+    switch( pConstraint->iColumn ){
+      case FSDIR_COLUMN_PATH: {
+        if( pConstraint->usable ){
+          idxPath = i;
+          seenPath = 0;
+        }else if( idxPath<0 ){
+          seenPath = 1;
+        }
+        break;
+      }
+      case FSDIR_COLUMN_DIR: {
+        if( pConstraint->usable ){
+          idxDir = i;
+          seenDir = 0;
+        }else if( idxDir<0 ){
+          seenDir = 1;
+        }
+        break;
+      }
+    } 
+  }
+  if( seenPath || seenDir ){
+    /* If input parameters are unusable, disallow this plan */
+    return SQLITE_CONSTRAINT;
  }

-  if( idx4<0 ){
+  if( idxPath<0 ){
    pIdxInfo->idxNum = 0;
-    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
+    /* The pIdxInfo->estimatedCost should have been initialized to a huge
+    ** number.  Leave it unchanged. */
+    pIdxInfo->estimatedRows = 0x7fffffff;
  }else{
-    pIdxInfo->aConstraintUsage[idx4].omit = 1;
-    pIdxInfo->aConstraintUsage[idx4].argvIndex = 1;
-    if( idx5>=0 ){
-      pIdxInfo->aConstraintUsage[idx5].omit = 1;
-      pIdxInfo->aConstraintUsage[idx5].argvIndex = 2;
+    pIdxInfo->aConstraintUsage[idxPath].omit = 1;
+    pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1;
+    if( idxDir>=0 ){
+      pIdxInfo->aConstraintUsage[idxDir].omit = 1;
+      pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2;
      pIdxInfo->idxNum = 2;
      pIdxInfo->estimatedCost = 10.0;
    }else{
@@ -791,7 +931,8 @@ static int fsdirRegister(sqlite3 *db){
    0,                         /* xRename */
    0,                         /* xSavepoint */
    0,                         /* xRelease */
-    0                          /* xRollbackTo */
+    0,                         /* xRollbackTo */
+    0,                         /* xShadowName */
  };

  int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
@@ -172,6 +172,7 @@ struct JsonParse {
  u8 nErr;           /* Number of errors seen */
  u16 iDepth;        /* Nesting depth */
  int nJson;         /* Length of the zJson string in bytes */
+  u32 iHold;         /* Replace cache line with the lowest iHold value */
 };

 /*
@@ -976,7 +977,8 @@ static int jsonParseFindParents(JsonParse *pParse){
 /*
 ** Magic number used for the JSON parse cache in sqlite3_get_auxdata()
 */
-#define JSON_CACHE_ID  (-429938)
+#define JSON_CACHE_ID  (-429938)  /* First cache entry */
+#define JSON_CACHE_SZ  4          /* Max number of cache entries */

 /*
 ** Obtain a complete parse of the JSON found in the first argument
@@ -988,16 +990,42 @@ static int jsonParseFindParents(JsonParse *pParse){
 */
 static JsonParse *jsonParseCached(
  sqlite3_context *pCtx,
-  sqlite3_value **argv
+  sqlite3_value **argv,
+  sqlite3_context *pErrCtx
 ){
  const char *zJson = (const char*)sqlite3_value_text(argv[0]);
  int nJson = sqlite3_value_bytes(argv[0]);
  JsonParse *p;
+  JsonParse *pMatch = 0;
+  int iKey;
+  int iMinKey = 0;
+  u32 iMinHold = 0xffffffff;
+  u32 iMaxHold = 0;
  if( zJson==0 ) return 0;
-  p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
-  if( p && p->nJson==nJson && memcmp(p->zJson,zJson,nJson)==0 ){
-    p->nErr = 0;
-    return p; /* The cached entry matches, so return it */
+  for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){
+    p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iKey);
+    if( p==0 ){
+      iMinKey = iKey;
+      break;
+    }
+    if( pMatch==0
+     && p->nJson==nJson
+     && memcmp(p->zJson,zJson,nJson)==0
+    ){
+      p->nErr = 0;
+      pMatch = p;
+    }else if( p->iHold<iMinHold ){
+      iMinHold = p->iHold;
+      iMinKey = iKey;
+    }
+    if( p->iHold>iMaxHold ){
+      iMaxHold = p->iHold;
+    }
+  }
+  if( pMatch ){
+    pMatch->nErr = 0;
+    pMatch->iHold = iMaxHold+1;
+    return pMatch;
  }
  p = sqlite3_malloc( sizeof(*p) + nJson + 1 );
  if( p==0 ){
@@ -1007,13 +1035,15 @@ static JsonParse *jsonParseCached(
  memset(p, 0, sizeof(*p));
  p->zJson = (char*)&p[1];
  memcpy((char*)p->zJson, zJson, nJson+1);
-  if( jsonParse(p, pCtx, p->zJson) ){
+  if( jsonParse(p, pErrCtx, p->zJson) ){
    sqlite3_free(p);
    return 0;
  }
  p->nJson = nJson;
-  sqlite3_set_auxdata(pCtx, JSON_CACHE_ID, p, (void(*)(void*))jsonParseFree);
-  return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
+  p->iHold = iMaxHold+1;
+  sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p,
+                      (void(*)(void*))jsonParseFree);
+  return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey);
 }

 /*
@@ -1386,7 +1416,7 @@ static void jsonArrayLengthFunc(
  u32 i;
  JsonNode *pNode;

-  p = jsonParseCached(ctx, argv);
+  p = jsonParseCached(ctx, argv, ctx);
  if( p==0 ) return;
  assert( p->nNode );
  if( argc==2 ){
@@ -1427,7 +1457,7 @@ static void jsonExtractFunc(
  int i;

  if( argc<2 ) return;
-  p = jsonParseCached(ctx, argv);
+  p = jsonParseCached(ctx, argv, ctx);
  if( p==0 ) return;
  jsonInit(&jx, ctx);
  jsonAppendChar(&jx, '[');
@@ -1734,22 +1764,21 @@ static void jsonTypeFunc(
  int argc,
  sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
+  JsonParse *p;          /* The parse */
  const char *zPath;
  JsonNode *pNode;

-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
-  assert( x.nNode );
+  p = jsonParseCached(ctx, argv, ctx);
+  if( p==0 ) return;
  if( argc==2 ){
    zPath = (const char*)sqlite3_value_text(argv[1]);
-    pNode = jsonLookup(&x, zPath, 0, ctx);
+    pNode = jsonLookup(p, zPath, 0, ctx);
  }else{
-    pNode = x.aNode;
+    pNode = p->aNode;
  }
  if( pNode ){
    sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC);
  }
-  jsonParseReset(&x);
 }

 /*
@@ -1763,15 +1792,10 @@ static void jsonValidFunc(
  int argc,
  sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
-  int rc = 0;
-
+  JsonParse *p;          /* The parse */
  UNUSED_PARAM(argc);
-  if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 ){
-    rc = 1;
-  }
-  jsonParseReset(&x);
-  sqlite3_result_int(ctx, rc);
+  p = jsonParseCached(ctx, argv, 0);
+  sqlite3_result_int(ctx, p!=0);
 }


@@ -1802,7 +1826,7 @@ static void jsonArrayStep(
    jsonAppendValue(pStr, argv[0]);
  }
 }
-static void jsonArrayFinal(sqlite3_context *ctx){
+static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){
  JsonString *pStr;
  pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
  if( pStr ){
@@ -1811,16 +1835,66 @@ static void jsonArrayFinal(sqlite3_context *ctx){
    if( pStr->bErr ){
      if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
      assert( pStr->bStatic );
-    }else{
-      sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
+    }else if( isFinal ){
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed,
                          pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
      pStr->bStatic = 1;
+    }else{
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
+      pStr->nUsed--;
    }
  }else{
    sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC);
  }
  sqlite3_result_subtype(ctx, JSON_SUBTYPE);
 }
+static void jsonArrayValue(sqlite3_context *ctx){
+  jsonArrayCompute(ctx, 0);
+}
+static void jsonArrayFinal(sqlite3_context *ctx){
+  jsonArrayCompute(ctx, 1);
+}
+
+#ifndef SQLITE_OMIT_WINDOWFUNC
+/*
+** This method works for both json_group_array() and json_group_object().
+** It works by removing the first element of the group by searching forward
+** to the first comma (",") that is not within a string and deleting all
+** text through that comma.
+*/
+static void jsonGroupInverse(
+  sqlite3_context *ctx,
+  int argc,
+  sqlite3_value **argv
+){
+  int i;
+  int inStr = 0;
+  char *z;
+  JsonString *pStr;
+  UNUSED_PARAM(argc);
+  UNUSED_PARAM(argv);
+  pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
+#ifdef NEVER
+  /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will
+  ** always have been called to initalize it */
+  if( NEVER(!pStr) ) return;
+#endif
+  z = pStr->zBuf;
+  for(i=1; z[i]!=',' || inStr; i++){
+    assert( i<pStr->nUsed );
+    if( z[i]=='"' ){
+      inStr = !inStr;
+    }else if( z[i]=='\\' ){
+      i++;
+    }
+  }
+  pStr->nUsed -= i;      
+  memmove(&z[1], &z[i+1], (size_t)pStr->nUsed-1);
+}
+#else
+# define jsonGroupInverse 0
+#endif
+

 /*
 ** json_group_obj(NAME,VALUE)
@@ -1852,7 +1926,7 @@ static void jsonObjectStep(
    jsonAppendValue(pStr, argv[1]);
  }
 }
-static void jsonObjectFinal(sqlite3_context *ctx){
+static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){
  JsonString *pStr;
  pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
  if( pStr ){
@@ -1860,16 +1934,26 @@ static void jsonObjectFinal(sqlite3_context *ctx){
    if( pStr->bErr ){
      if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
      assert( pStr->bStatic );
-    }else{
-      sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
+    }else if( isFinal ){
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed,
                          pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
      pStr->bStatic = 1;
+    }else{
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
+      pStr->nUsed--;
    }
  }else{
    sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC);
  }
  sqlite3_result_subtype(ctx, JSON_SUBTYPE);
 }
+static void jsonObjectValue(sqlite3_context *ctx){
+  jsonObjectCompute(ctx, 0);
+}
+static void jsonObjectFinal(sqlite3_context *ctx){
+  jsonObjectCompute(ctx, 1);
+}
+


 #ifndef SQLITE_OMIT_VIRTUALTABLE
@@ -1910,6 +1994,9 @@ static int jsonEachConnect(
 #define JEACH_PARENT  5
 #define JEACH_FULLKEY 6
 #define JEACH_PATH    7
+/* The xBestIndex method assumes that the JSON and ROOT columns are
+** the last two columns in the table.  Should this ever changes, be
+** sure to update the xBestIndex method. */
 #define JEACH_JSON    8
 #define JEACH_ROOT    9

@@ -2118,7 +2205,7 @@ static int jsonEachColumn(
        }
        if( p->eType==JSON_ARRAY ){
          jsonPrintf(30, &x, "[%d]", p->iRowid);
-        }else{
+        }else if( p->eType==JSON_OBJECT ){
          jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1);
        }
      }
@@ -2167,35 +2254,54 @@ static int jsonEachBestIndex(
  sqlite3_vtab *tab,
  sqlite3_index_info *pIdxInfo
 ){
-  int i;
-  int jsonIdx = -1;
-  int rootIdx = -1;
+  int i;                     /* Loop counter or computed array index */
+  int aIdx[2];               /* Index of constraints for JSON and ROOT */
+  int unusableMask = 0;      /* Mask of unusable JSON and ROOT constraints */
+  int idxMask = 0;           /* Mask of usable == constraints JSON and ROOT */
  const struct sqlite3_index_constraint *pConstraint;

+  /* This implementation assumes that JSON and ROOT are the last two
+  ** columns in the table */
+  assert( JEACH_ROOT == JEACH_JSON+1 );
  UNUSED_PARAM(tab);
+  aIdx[0] = aIdx[1] = -1;
  pConstraint = pIdxInfo->aConstraint;
  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->usable==0 ) continue;
-    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    switch( pConstraint->iColumn ){
-      case JEACH_JSON:   jsonIdx = i;    break;
-      case JEACH_ROOT:   rootIdx = i;    break;
-      default:           /* no-op */     break;
+    int iCol;
+    int iMask;
+    if( pConstraint->iColumn < JEACH_JSON ) continue;
+    iCol = pConstraint->iColumn - JEACH_JSON;
+    assert( iCol==0 || iCol==1 );
+    iMask = 1 << iCol;
+    if( pConstraint->usable==0 ){
+      unusableMask |= iMask;
+    }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      aIdx[iCol] = i;
+      idxMask |= iMask;
    }
  }
-  if( jsonIdx<0 ){
+  if( (unusableMask & ~idxMask)!=0 ){
+    /* If there are any unusable constraints on JSON or ROOT, then reject
+    ** this entire plan */
+    return SQLITE_CONSTRAINT;
+  }
+  if( aIdx[0]<0 ){
+    /* No JSON input.  Leave estimatedCost at the huge value that it was
+    ** initialized to to discourage the query planner from selecting this
+    ** plan. */
    pIdxInfo->idxNum = 0;
-    pIdxInfo->estimatedCost = 1e99;
  }else{
    pIdxInfo->estimatedCost = 1.0;
-    pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1;
-    pIdxInfo->aConstraintUsage[jsonIdx].omit = 1;
-    if( rootIdx<0 ){
-      pIdxInfo->idxNum = 1;
+    i = aIdx[0];
+    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[i].omit = 1;
+    if( aIdx[1]<0 ){
+      pIdxInfo->idxNum = 1;  /* Only JSON supplied.  Plan 1 */
    }else{
-      pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2;
-      pIdxInfo->aConstraintUsage[rootIdx].omit = 1;
-      pIdxInfo->idxNum = 3;
+      i = aIdx[1];
+      pIdxInfo->aConstraintUsage[i].argvIndex = 2;
+      pIdxInfo->aConstraintUsage[i].omit = 1;
+      pIdxInfo->idxNum = 3;  /* Both JSON and ROOT are supplied.  Plan 3 */
    }
  }
  return SQLITE_OK;
@@ -2304,7 +2410,8 @@ static sqlite3_module jsonEachModule = {
  0,                         /* xRename */
  0,                         /* xSavepoint */
  0,                         /* xRelease */
-  0                          /* xRollbackTo */
+  0,                         /* xRollbackTo */
+  0                          /* xShadowName */
 };

 /* The methods of the json_tree virtual table. */
@@ -2331,7 +2438,8 @@ static sqlite3_module jsonTreeModule = {
  0,                         /* xRename */
  0,                         /* xSavepoint */
  0,                         /* xRelease */
-  0                          /* xRollbackTo */
+  0,                         /* xRollbackTo */
+  0                          /* xShadowName */
 };
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

@@ -2377,9 +2485,12 @@ int sqlite3Json1Init(sqlite3 *db){
     int nArg;
     void (*xStep)(sqlite3_context*,int,sqlite3_value**);
     void (*xFinal)(sqlite3_context*);
+     void (*xValue)(sqlite3_context*);
  } aAgg[] = {
-    { "json_group_array",     1,   jsonArrayStep,   jsonArrayFinal  },
-    { "json_group_object",    2,   jsonObjectStep,  jsonObjectFinal },
+    { "json_group_array",     1,
+      jsonArrayStep,   jsonArrayFinal,  jsonArrayValue  },
+    { "json_group_object",    2,
+      jsonObjectStep,  jsonObjectFinal, jsonObjectValue },
  };
 #ifndef SQLITE_OMIT_VIRTUALTABLE
  static const struct {
@@ -2396,11 +2507,14 @@ int sqlite3Json1Init(sqlite3 *db){
                                 (void*)&aFunc[i].flag,
                                 aFunc[i].xFunc, 0, 0);
  }
+#ifndef SQLITE_OMIT_WINDOWFUNC
  for(i=0; i<sizeof(aAgg)/sizeof(aAgg[0]) && rc==SQLITE_OK; i++){
-    rc = sqlite3_create_function(db, aAgg[i].zName, aAgg[i].nArg,
+    rc = sqlite3_create_window_function(db, aAgg[i].zName, aAgg[i].nArg,
                                 SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0,
-                                 0, aAgg[i].xStep, aAgg[i].xFinal);
+                                 aAgg[i].xStep, aAgg[i].xFinal,
+                                 aAgg[i].xValue, jsonGroupInverse, 0);
  }
+#endif
 #ifndef SQLITE_OMIT_VIRTUALTABLE
  for(i=0; i<sizeof(aMod)/sizeof(aMod[0]) && rc==SQLITE_OK; i++){
    rc = sqlite3_create_module(db, aMod[i].zName, aMod[i].pModule, 0);
@@ -0,0 +1,428 @@
+/*
+** 2018-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file demonstrates an eponymous virtual table that returns information
+** from sqlite3_status64() and sqlite3_db_status().
+**
+** Usage example:
+**
+**     .load ./memstat
+**     .mode quote
+**     .header on
+**     SELECT * FROM memstat;
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB)
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* memstat_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a memstat virtual table
+*/
+typedef struct memstat_vtab memstat_vtab;
+struct memstat_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  sqlite3 *db;        /* Database connection for this memstat vtab */
+};
+
+/* memstat_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct memstat_cursor memstat_cursor;
+struct memstat_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  sqlite3 *db;               /* Database connection for this cursor */
+  int iRowid;                /* Current row in aMemstatColumn[] */
+  int iDb;                   /* Which schema we are looking at */
+  int nDb;                   /* Number of schemas */
+  char **azDb;               /* Names of all schemas */
+  sqlite3_int64 aVal[2];     /* Result values */
+};
+
+/*
+** The memstatConnect() method is invoked to create a new
+** memstat_vtab that describes the memstat virtual table.
+**
+** Think of this routine as the constructor for memstat_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the memstat_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against memstat will look like.
+*/
+static int memstatConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  memstat_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define MSV_COLUMN_NAME    0   /* Name of quantity being measured */
+#define MSV_COLUMN_SCHEMA  1   /* schema name */
+#define MSV_COLUMN_VALUE   2   /* Current value */
+#define MSV_COLUMN_HIWTR   3   /* Highwater mark */
+
+  rc = sqlite3_declare_vtab(db,"CREATE TABLE x(name,schema,value,hiwtr)");
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->db = db;
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for memstat_cursor objects.
+*/
+static int memstatDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new memstat_cursor object.
+*/
+static int memstatOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  memstat_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->db = ((memstat_vtab*)p)->db;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Clear all the schema names from a cursor
+*/
+static void memstatClearSchema(memstat_cursor *pCur){
+  int i;
+  if( pCur->azDb==0 ) return;
+  for(i=0; i<pCur->nDb; i++){
+    sqlite3_free(pCur->azDb[i]);
+  }
+  sqlite3_free(pCur->azDb);
+  pCur->azDb = 0;
+  pCur->nDb = 0;
+}
+
+/*
+** Fill in the azDb[] array for the cursor.
+*/
+static int memstatFindSchemas(memstat_cursor *pCur){
+  sqlite3_stmt *pStmt = 0;
+  int rc;
+  if( pCur->nDb ) return SQLITE_OK;
+  rc = sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pStmt, 0);
+  if( rc ){
+    sqlite3_finalize(pStmt);
+    return rc;
+  }
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    char **az, *z;
+    az = sqlite3_realloc(pCur->azDb, sizeof(char*)*(pCur->nDb+1));
+    if( az==0 ){
+      memstatClearSchema(pCur);
+      return SQLITE_NOMEM;
+    }
+    pCur->azDb = az;
+    z = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
+    if( z==0 ){
+      memstatClearSchema(pCur);
+      return SQLITE_NOMEM;
+    }
+    pCur->azDb[pCur->nDb] = z;
+    pCur->nDb++;
+  }
+  sqlite3_finalize(pStmt);
+  return SQLITE_OK;
+}
+
+
+/*
+** Destructor for a memstat_cursor.
+*/
+static int memstatClose(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  memstatClearSchema(pCur);
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Allowed values for aMemstatColumn[].eType
+*/
+#define MSV_GSTAT   0          /* sqlite3_status64() information */
+#define MSV_DB      1          /* sqlite3_db_status() information */
+#define MSV_ZIPVFS  2          /* ZIPVFS file-control with 64-bit return */
+
+/*
+** An array of quantities that can be measured and reported by
+** this virtual table
+*/
+static const struct MemstatColumns {
+  const char *zName;    /* Symbolic name */
+  unsigned char eType;  /* Type of interface */
+  unsigned char mNull;  /* Bitmask of which columns are NULL */
+                        /* 2: dbname,  4: current,  8: hiwtr */
+  int eOp;              /* Opcode */
+} aMemstatColumn[] = {
+ {"MEMORY_USED",            MSV_GSTAT,  2, SQLITE_STATUS_MEMORY_USED          },
+ {"MALLOC_SIZE",            MSV_GSTAT,  6, SQLITE_STATUS_MALLOC_SIZE          },
+ {"MALLOC_COUNT",           MSV_GSTAT,  2, SQLITE_STATUS_MALLOC_COUNT         },
+ {"PAGECACHE_USED",         MSV_GSTAT,  2, SQLITE_STATUS_PAGECACHE_USED       },
+ {"PAGECACHE_OVERFLOW",     MSV_GSTAT,  2, SQLITE_STATUS_PAGECACHE_OVERFLOW   },
+ {"PAGECACHE_SIZE",         MSV_GSTAT,  6, SQLITE_STATUS_PAGECACHE_SIZE       },
+ {"PARSER_STACK",           MSV_GSTAT,  6, SQLITE_STATUS_PARSER_STACK         },
+ {"DB_LOOKASIDE_USED",      MSV_DB,     2, SQLITE_DBSTATUS_LOOKASIDE_USED     },
+ {"DB_LOOKASIDE_HIT",       MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_HIT      },
+ {"DB_LOOKASIDE_MISS_SIZE", MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE},
+ {"DB_LOOKASIDE_MISS_FULL", MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL},
+ {"DB_CACHE_USED",          MSV_DB,    10, SQLITE_DBSTATUS_CACHE_USED         },
+#if SQLITE_VERSION_NUMBER >= 3140000
+ {"DB_CACHE_USED_SHARED",   MSV_DB,    10, SQLITE_DBSTATUS_CACHE_USED_SHARED  },
+#endif
+ {"DB_SCHEMA_USED",         MSV_DB,    10, SQLITE_DBSTATUS_SCHEMA_USED        },
+ {"DB_STMT_USED",           MSV_DB,    10, SQLITE_DBSTATUS_STMT_USED          },
+ {"DB_CACHE_HIT",           MSV_DB,    10, SQLITE_DBSTATUS_CACHE_HIT          },
+ {"DB_CACHE_MISS",          MSV_DB,    10, SQLITE_DBSTATUS_CACHE_MISS         },
+ {"DB_CACHE_WRITE",         MSV_DB,    10, SQLITE_DBSTATUS_CACHE_WRITE        },
+#if SQLITE_VERSION_NUMBER >= 3230000
+ {"DB_CACHE_SPILL",         MSV_DB,    10, SQLITE_DBSTATUS_CACHE_SPILL        },
+#endif
+ {"DB_DEFERRED_FKS",        MSV_DB,    10, SQLITE_DBSTATUS_DEFERRED_FKS       },
+#ifdef SQLITE_ENABLE_ZIPVFS
+ {"ZIPVFS_CACHE_USED",      MSV_ZIPVFS, 8, 231454 },
+ {"ZIPVFS_CACHE_HIT",       MSV_ZIPVFS, 8, 231455 },
+ {"ZIPVFS_CACHE_MISS",      MSV_ZIPVFS, 8, 231456 },
+ {"ZIPVFS_CACHE_WRITE",     MSV_ZIPVFS, 8, 231457 },
+ {"ZIPVFS_DIRECT_READ",     MSV_ZIPVFS, 8, 231458 },
+ {"ZIPVFS_DIRECT_BYTES",    MSV_ZIPVFS, 8, 231459 },
+#endif /* SQLITE_ENABLE_ZIPVFS */
+};
+#define MSV_NROW (sizeof(aMemstatColumn)/sizeof(aMemstatColumn[0]))
+
+/*
+** Advance a memstat_cursor to its next row of output.
+*/
+static int memstatNext(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  int i;
+  assert( pCur->iRowid<=MSV_NROW );
+  while(1){
+    i = (int)pCur->iRowid - 1;
+    if( i<0 || (aMemstatColumn[i].mNull & 2)!=0 || (++pCur->iDb)>=pCur->nDb ){
+      pCur->iRowid++;
+      if( pCur->iRowid>MSV_NROW ) return SQLITE_OK;  /* End of the table */
+      pCur->iDb = 0;
+      i++;
+    }
+    pCur->aVal[0] = 0;
+    pCur->aVal[1] = 0;    
+    switch( aMemstatColumn[i].eType ){
+      case MSV_GSTAT: {
+        if( sqlite3_libversion_number()>=3010000 ){
+          sqlite3_status64(aMemstatColumn[i].eOp,
+                           &pCur->aVal[0], &pCur->aVal[1],0);
+        }else{
+          int xCur, xHiwtr;
+          sqlite3_status(aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0);
+          pCur->aVal[0] = xCur;
+          pCur->aVal[1] = xHiwtr;
+        }
+        break;
+      }
+      case MSV_DB: {
+        int xCur, xHiwtr;
+        sqlite3_db_status(pCur->db, aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0);
+        pCur->aVal[0] = xCur;
+        pCur->aVal[1] = xHiwtr;
+        break;
+      }
+      case MSV_ZIPVFS: {
+        int rc;
+        rc = sqlite3_file_control(pCur->db, pCur->azDb[pCur->iDb],
+                                  aMemstatColumn[i].eOp, (void*)&pCur->aVal[0]);
+        if( rc!=SQLITE_OK ) continue;
+        break;
+      }
+    }
+    break;
+  }
+  return SQLITE_OK;
+}
+  
+
+/*
+** Return values of columns for the row at which the memstat_cursor
+** is currently pointing.
+*/
+static int memstatColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int iCol                    /* Which column to return */
+){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  int i;
+  assert( pCur->iRowid>0 && pCur->iRowid<=MSV_NROW );
+  i = (int)pCur->iRowid - 1;
+  if( (aMemstatColumn[i].mNull & (1<<iCol))!=0 ){
+    return SQLITE_OK;
+  }
+  switch( iCol ){
+    case MSV_COLUMN_NAME: {
+      sqlite3_result_text(ctx, aMemstatColumn[i].zName, -1, SQLITE_STATIC);
+      break;
+    }
+    case MSV_COLUMN_SCHEMA: {
+      sqlite3_result_text(ctx, pCur->azDb[pCur->iDb], -1, 0);
+      break;
+    }
+    case MSV_COLUMN_VALUE: {
+      sqlite3_result_int64(ctx, pCur->aVal[0]);
+      break;
+    }
+    case MSV_COLUMN_HIWTR: {
+      sqlite3_result_int64(ctx, pCur->aVal[1]);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int memstatRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  *pRowid = pCur->iRowid*1000 + pCur->iDb;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int memstatEof(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  return pCur->iRowid>MSV_NROW;
+}
+
+/*
+** This method is called to "rewind" the memstat_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to memstatColumn() or memstatRowid() or 
+** memstatEof().
+*/
+static int memstatFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  memstat_cursor *pCur = (memstat_cursor *)pVtabCursor;
+  int rc = memstatFindSchemas(pCur);
+  if( rc ) return rc;
+  pCur->iRowid = 0;
+  pCur->iDb = 0;
+  return memstatNext(pVtabCursor);
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the memstat virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int memstatBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = (double)500;
+  pIdxInfo->estimatedRows = 500;
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** memstat virtual table.
+*/
+static sqlite3_module memstatModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  memstatConnect,            /* xConnect */
+  memstatBestIndex,          /* xBestIndex */
+  memstatDisconnect,         /* xDisconnect */
+  0,                         /* xDestroy */
+  memstatOpen,               /* xOpen - open a cursor */
+  memstatClose,              /* xClose - close a cursor */
+  memstatFilter,             /* xFilter - configure scan constraints */
+  memstatNext,               /* xNext - advance a cursor */
+  memstatEof,                /* xEof - check for end of scan */
+  memstatColumn,             /* xColumn - read data */
+  memstatRowid,              /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+int sqlite3MemstatVtabInit(sqlite3 *db){
+  int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3_create_module(db, "sqlite_memstat", &memstatModule, 0);
+#endif
+  return rc;
+}
+
+#ifndef SQLITE_CORE
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_memstat_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3MemstatVtabInit(db);
+#endif
+  return rc;
+}
+#endif /* SQLITE_CORE */
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB) */
@@ -593,7 +593,7 @@ char *sqlite3_normalize(const char *zSql){
    }
  }
  while( j>0 && z[j-1]==' ' ){ j--; }
-  if( i>0 && z[j-1]!=';' ){ z[j++] = ';'; }
+  if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
  z[j] = 0;

  /* Make a second pass converting "in(...)" where the "..." is not a
@@ -270,6 +270,15 @@ static int seriesFilter(
  }else{
    pCur->iStep = 1;
  }
+  for(i=0; i<argc; i++){
+    if( sqlite3_value_type(argv[i])==SQLITE_NULL ){
+      /* If any of the constraints have a NULL value, then return no rows.
+      ** See ticket https://www.sqlite.org/src/info/fac496b61722daf2 */
+      pCur->mnValue = 1;
+      pCur->mxValue = 0;
+      break;
+    }
+  }
  if( idxNum & 8 ){
    pCur->isDesc = 1;
    pCur->iValue = pCur->mxValue;
@@ -304,44 +313,45 @@ static int seriesBestIndex(
  sqlite3_vtab *tab,
  sqlite3_index_info *pIdxInfo
 ){
-  int i;                 /* Loop over constraints */
+  int i, j;              /* Loop over constraints */
  int idxNum = 0;        /* The query plan bitmask */
-  int startIdx = -1;     /* Index of the start= constraint, or -1 if none */
-  int stopIdx = -1;      /* Index of the stop= constraint, or -1 if none */
-  int stepIdx = -1;      /* Index of the step= constraint, or -1 if none */
+  int unusableMask = 0;  /* Mask of unusable constraints */
  int nArg = 0;          /* Number of arguments that seriesFilter() expects */
-
+  int aIdx[3];           /* Constraints on start, stop, and step */
  const struct sqlite3_index_constraint *pConstraint;
+
+  /* This implementation assumes that the start, stop, and step columns
+  ** are the last three columns in the virtual table. */
+  assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 );
+  assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 );
+  aIdx[0] = aIdx[1] = aIdx[2] = -1;
  pConstraint = pIdxInfo->aConstraint;
  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->usable==0 ) continue;
-    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    switch( pConstraint->iColumn ){
-      case SERIES_COLUMN_START:
-        startIdx = i;
-        idxNum |= 1;
-        break;
-      case SERIES_COLUMN_STOP:
-        stopIdx = i;
-        idxNum |= 2;
-        break;
-      case SERIES_COLUMN_STEP:
-        stepIdx = i;
-        idxNum |= 4;
-        break;
+    int iCol;    /* 0 for start, 1 for stop, 2 for step */
+    int iMask;   /* bitmask for those column */
+    if( pConstraint->iColumn<SERIES_COLUMN_START ) continue;
+    iCol = pConstraint->iColumn - SERIES_COLUMN_START;
+    assert( iCol>=0 && iCol<=2 );
+    iMask = 1 << iCol;
+    if( pConstraint->usable==0 ){
+      unusableMask |=  iMask;
+      continue;
+    }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      idxNum |= iMask;
+      aIdx[iCol] = i;
    }
  }
-  if( startIdx>=0 ){
-    pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY;
+  for(i=0; i<3; i++){
+    if( (j = aIdx[i])>=0 ){
+      pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg;
+      pIdxInfo->aConstraintUsage[j].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+    }
  }
-  if( stopIdx>=0 ){
-    pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
-  }
-  if( stepIdx>=0 ){
-    pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+  if( (unusableMask & ~idxNum)!=0 ){
+    /* The start, stop, and step columns are inputs.  Therefore if there
+    ** are unusable constraints on any of start, stop, or step then
+    ** this plan is unusable */
+    return SQLITE_CONSTRAINT;
  }
  if( (idxNum & 3)==3 ){
    /* Both start= and stop= boundaries are available.  This is the 
@@ -356,7 +366,6 @@ static int seriesBestIndex(
    /* If either boundary is missing, we have to generate a huge span
    ** of numbers.  Make this case very expensive so that the query
    ** planner will work hard to avoid it. */
-    pIdxInfo->estimatedCost = (double)2147483647;
    pIdxInfo->estimatedRows = 2147483647;
  }
  pIdxInfo->idxNum = idxNum;
@@ -10,7 +10,7 @@
 **
 ******************************************************************************
 **
-** This SQLite extension implements a functions that compute SHA1 hashes.
+** This SQLite extension implements functions that compute SHA1 hashes.
 ** Two SQL functions are implemented:
 **
 **     sha1(X)
@@ -10,7 +10,7 @@
 **
 ******************************************************************************
 **
-** This SQLite extension implements a functions that compute SHA1 hashes.
+** This SQLite extension implements functions that compute SHA3 hashes.
 ** Two SQL functions are implemented:
 **
 **     sha3(X,SIZE)
@@ -764,7 +764,7 @@ static int editDist3ConfigLoad(
    assert( zTo!=0 || nTo==0 );
    if( nFrom>100 || nTo>100 ) continue;
    if( iCost<0 ) continue;
-    if( iCost>10000 ) continue;   /* Costs above 10K are considered infinite */
+    if( iCost>=10000 ) continue;  /* Costs above 10K are considered infinite */
    if( pLang==0 || iLang!=iLangPrev ){
      EditDist3Lang *pNew;
      pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0]));
@@ -835,6 +835,7 @@ static int utf8Len(unsigned char c, int N){
 ** the given string.
 */
 static int matchTo(EditDist3Cost *p, const char *z, int n){
+  assert( n>0 );
  if( p->a[p->nFrom]!=z[0] ) return 0;
  if( p->nTo>n ) return 0;
  if( strncmp(p->a+p->nFrom, z, p->nTo)!=0 ) return 0;
@@ -847,8 +848,10 @@ static int matchTo(EditDist3Cost *p, const char *z, int n){
 */
 static int matchFrom(EditDist3Cost *p, const char *z, int n){
  assert( p->nFrom<=n );
-  if( p->a[0]!=z[0] ) return 0;
-  if( strncmp(p->a, z, p->nFrom)!=0 ) return 0;
+  if( p->nFrom ){
+    if( p->a[0]!=z[0] ) return 0;
+    if( strncmp(p->a, z, p->nFrom)!=0 ) return 0;
+  }
  return 1;
 }

@@ -864,6 +867,7 @@ static int matchFromTo(
 ){
  int b1 = pStr->a[n1].nByte;
  if( b1>n2 ) return 0;
+  assert( b1>0 );
  if( pStr->z[n1]!=z2[0] ) return 0;
  if( strncmp(pStr->z+n1, z2, b1)!=0 ) return 0;
  return 1;
@@ -1291,6 +1295,9 @@ typedef struct Transliteration Transliteration;
 struct Transliteration {
 unsigned short int cFrom;
 unsigned char cTo0, cTo1, cTo2, cTo3;
+#ifdef SQLITE_SPELLFIX_5BYTE_MAPPINGS
+ unsigned char cTo4;
+#endif
 };

 /*
@@ -1704,7 +1711,11 @@ static const Transliteration *spellfixFindTranslit(int c, int *pxTop){
 ** should be freed by the caller.
 */
 static unsigned char *transliterate(const unsigned char *zIn, int nIn){
+#ifdef SQLITE_SPELLFIX_5BYTE_MAPPINGS
+  unsigned char *zOut = sqlite3_malloc64( nIn*5 + 1 );
+#else
  unsigned char *zOut = sqlite3_malloc64( nIn*4 + 1 );
+#endif
  int c, sz, nOut;
  if( zOut==0 ) return 0;
  nOut = 0;
@@ -1728,6 +1739,11 @@ static unsigned char *transliterate(const unsigned char *zIn, int nIn){
              zOut[nOut++] = tbl[x].cTo2;
              if( tbl[x].cTo3 ){
                zOut[nOut++] = tbl[x].cTo3;
+#ifdef SQLITE_SPELLFIX_5BYTE_MAPPINGS
+                if( tbl[x].cTo4 ){
+                  zOut[nOut++] = tbl[x].cTo4;
+                }
+#endif /* SQLITE_SPELLFIX_5BYTE_MAPPINGS */
              }
            }
          }
@@ -266,6 +266,7 @@ static sqlite3_module stmtModule = {
  0,                         /* xSavepoint */
  0,                         /* xRelease */
  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
 };

 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -0,0 +1,268 @@
+/*
+** 2018-04-19
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file implements a template virtual-table.
+** Developers can make a copy of this file as a baseline for writing
+** new virtual tables and/or table-valued functions.
+**
+** Steps for writing a new virtual table implementation:
+**
+**     (1)  Make a copy of this file.  Perhaps call it "mynewvtab.c"
+**
+**     (2)  Replace this header comment with something appropriate for
+**          the new virtual table
+**
+**     (3)  Change every occurrence of "templatevtab" to some other string
+**          appropriate for the new virtual table.  Ideally, the new string
+**          should be the basename of the source file: "mynewvtab".  Also
+**          globally change "TEMPLATEVTAB" to "MYNEWVTAB".
+**
+**     (4)  Run a test compilation to make sure the unmodified virtual
+**          table works.
+**
+**     (5)  Begin making incremental changes, testing as you go, to evolve
+**          the new virtual table to do what you want it to do.
+**
+** This template is minimal, in the sense that it uses only the required
+** methods on the sqlite3_module object.  As a result, templatevtab is
+** a read-only and eponymous-only table.  Those limitation can be removed
+** by adding new methods.
+**
+** This template implements an eponymous-only virtual table with a rowid and
+** two columns named "a" and "b".  The table as 10 rows with fixed integer
+** values. Usage example:
+**
+**     SELECT rowid, a, b FROM templatevtab;
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+
+/* templatevtab_vtab is a subclass of sqlite3_vtab which is
+** underlying representation of the virtual table
+*/
+typedef struct templatevtab_vtab templatevtab_vtab;
+struct templatevtab_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  /* Add new fields here, as necessary */
+};
+
+/* templatevtab_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct templatevtab_cursor templatevtab_cursor;
+struct templatevtab_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  /* Insert new fields here.  For this templatevtab we only keep track
+  ** of the rowid */
+  sqlite3_int64 iRowid;      /* The rowid */
+};
+
+/*
+** The templatevtabConnect() method is invoked to create a new
+** template virtual table.
+**
+** Think of this routine as the constructor for templatevtab_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the templatevtab_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against the virtual table will look like.
+*/
+static int templatevtabConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  templatevtab_vtab *pNew;
+  int rc;
+
+  rc = sqlite3_declare_vtab(db,
+           "CREATE TABLE x(a,b)"
+       );
+  /* For convenience, define symbolic names for the index to each column. */
+#define TEMPLATEVTAB_A  0
+#define TEMPLATEVTAB_B  1
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for templatevtab_vtab objects.
+*/
+static int templatevtabDisconnect(sqlite3_vtab *pVtab){
+  templatevtab_vtab *p = (templatevtab_vtab*)pVtab;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new templatevtab_cursor object.
+*/
+static int templatevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  templatevtab_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a templatevtab_cursor.
+*/
+static int templatevtabClose(sqlite3_vtab_cursor *cur){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a templatevtab_cursor to its next row of output.
+*/
+static int templatevtabNext(sqlite3_vtab_cursor *cur){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  pCur->iRowid++;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the templatevtab_cursor
+** is currently pointing.
+*/
+static int templatevtabColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  switch( i ){
+    case TEMPLATEVTAB_A:
+      sqlite3_result_int(ctx, 1000 + pCur->iRowid);
+      break;
+    default:
+      assert( i==TEMPLATEVTAB_B );
+      sqlite3_result_int(ctx, 2000 + pCur->iRowid);
+      break;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int templatevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int templatevtabEof(sqlite3_vtab_cursor *cur){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  return pCur->iRowid>=10;
+}
+
+/*
+** This method is called to "rewind" the templatevtab_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to templatevtabColumn() or templatevtabRowid() or 
+** templatevtabEof().
+*/
+static int templatevtabFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  templatevtab_cursor *pCur = (templatevtab_cursor *)pVtabCursor;
+  pCur->iRowid = 1;
+  return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int templatevtabBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = (double)10;
+  pIdxInfo->estimatedRows = 10;
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** virtual table.
+*/
+static sqlite3_module templatevtabModule = {
+  /* iVersion    */ 0,
+  /* xCreate     */ 0,
+  /* xConnect    */ templatevtabConnect,
+  /* xBestIndex  */ templatevtabBestIndex,
+  /* xDisconnect */ templatevtabDisconnect,
+  /* xDestroy    */ 0,
+  /* xOpen       */ templatevtabOpen,
+  /* xClose      */ templatevtabClose,
+  /* xFilter     */ templatevtabFilter,
+  /* xNext       */ templatevtabNext,
+  /* xEof        */ templatevtabEof,
+  /* xColumn     */ templatevtabColumn,
+  /* xRowid      */ templatevtabRowid,
+  /* xUpdate     */ 0,
+  /* xBegin      */ 0,
+  /* xSync       */ 0,
+  /* xCommit     */ 0,
+  /* xRollback   */ 0,
+  /* xFindMethod */ 0,
+  /* xRename     */ 0,
+  /* xSavepoint  */ 0,
+  /* xRelease    */ 0,
+  /* xRollbackTo */ 0,
+  /* xShadowName */ 0
+};
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_templatevtab_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  rc = sqlite3_create_module(db, "templatevtab", &templatevtabModule, 0);
+  return rc;
+}
@@ -1350,7 +1350,8 @@ static int createUnionVtab(sqlite3 *db){
    0,                            /* xRename */
    0,                            /* xSavepoint */
    0,                            /* xRelease */
-    0                             /* xRollbackTo */
+    0,                            /* xRollbackTo */
+    0                             /* xShadowName */
  };
  int rc;

@@ -492,6 +492,7 @@ static sqlite3_module vtablogModule = {
  0,                         /* xSavepoint */
  0,                         /* xRelease */
  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
 };

 #ifdef _WIN32
@@ -1296,25 +1296,26 @@ static int zipfileBestIndex(
  sqlite3_index_info *pIdxInfo
 ){
  int i;
+  int idx = -1;
+  int unusable = 0;

  for(i=0; i<pIdxInfo->nConstraint; i++){
    const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
-    if( pCons->usable==0 ) continue;
-    if( pCons->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
    if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue;
-    break;
+    if( pCons->usable==0 ){
+      unusable = 1;
+    }else if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      idx = i;
+    }
  }
-
-  if( i<pIdxInfo->nConstraint ){
-    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
-    pIdxInfo->aConstraintUsage[i].omit = 1;
+  if( idx>=0 ){
+    pIdxInfo->aConstraintUsage[idx].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[idx].omit = 1;
    pIdxInfo->estimatedCost = 1000.0;
    pIdxInfo->idxNum = 1;
-  }else{
-    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
-    pIdxInfo->idxNum = 0;
+  }else if( unusable ){
+    return SQLITE_CONSTRAINT;
  }
-
  return SQLITE_OK;
 }

@@ -1500,6 +1501,19 @@ static u32 zipfileGetTime(sqlite3_value *pVal){
  return (u32)sqlite3_value_int64(pVal);
 }

+/*
+** Unless it is NULL, entry pOld is currently part of the pTab->pFirstEntry
+** linked list.  Remove it from the list and free the object.
+*/
+static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){
+  if( pOld ){
+    ZipfileEntry **pp;
+    for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext));
+    *pp = (*pp)->pNext;
+    zipfileEntryFree(pOld);
+  }
+}
+
 /*
 ** xUpdate method.
 */
@@ -1524,6 +1538,8 @@ static int zipfileUpdate(
  u8 *pFree = 0;                  /* Free this */
  char *zFree = 0;                /* Also free this */
  ZipfileEntry *pOld = 0;
+  ZipfileEntry *pOld2 = 0;
+  int bUpdate = 0;                /* True for an update that modifies "name" */
  int bIsDir = 0;
  u32 iCrc32 = 0;

@@ -1536,6 +1552,12 @@ static int zipfileUpdate(
  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
    int nDelete = (int)strlen(zDelete);
+    if( nVal>1 ){
+      const char *zUpdate = (const char*)sqlite3_value_text(apVal[1]);
+      if( zUpdate && zipfileComparePath(zUpdate, zDelete, nDelete)!=0 ){
+        bUpdate = 1;
+      }
+    }
    for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){
      if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){
        break;
@@ -1613,8 +1635,9 @@ static int zipfileUpdate(
      }
    }

-    /* Check that we're not inserting a duplicate entry */
-    if( pOld==0 && rc==SQLITE_OK ){
+    /* Check that we're not inserting a duplicate entry -OR- updating an
+    ** entry with a path, thereby making it into a duplicate. */
+    if( (pOld==0 || bUpdate) && rc==SQLITE_OK ){
      ZipfileEntry *p;
      for(p=pTab->pFirstEntry; p; p=p->pNext){
        if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){
@@ -1623,7 +1646,7 @@ static int zipfileUpdate(
              goto zipfile_update_done;
            }
            case SQLITE_REPLACE: {
-              pOld = p;
+              pOld2 = p;
              break;
            }
            default: {
@@ -1661,18 +1684,17 @@ static int zipfileUpdate(
    }
  }

-  if( rc==SQLITE_OK && pOld ){
-    ZipfileEntry **pp;
+  if( rc==SQLITE_OK && (pOld || pOld2) ){
    ZipfileCsr *pCsr;
    for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){
-      if( pCsr->pCurrent==pOld ){
-        pCsr->pCurrent = pOld->pNext;
+      if( pCsr->pCurrent && (pCsr->pCurrent==pOld || pCsr->pCurrent==pOld2) ){
+        pCsr->pCurrent = pCsr->pCurrent->pNext;
        pCsr->bNoop = 1;
      }
    }
-    for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext));
-    *pp = (*pp)->pNext;
-    zipfileEntryFree(pOld);
+
+    zipfileRemoveEntryFromList(pTab, pOld);
+    zipfileRemoveEntryFromList(pTab, pOld2);
  }

 zipfile_update_done:
@@ -2040,9 +2062,11 @@ void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
  p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]);

  /* Append the data to the body of the new archive */
-  if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out;
-  memcpy(&p->body.a[p->body.n], aData, nData);
-  p->body.n += nData;
+  if( nData>0 ){
+    if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out;
+    memcpy(&p->body.a[p->body.n], aData, nData);
+    p->body.n += nData;
+  }

  /* Append the CDS record to the directory of the new archive */
  nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9;
@@ -29,7 +29,9 @@ void usage(const char *zArgv0){
 "Where options are:\n"
 "\n"
 "    -step NSTEP\n"
+"    -statstep NSTATSTEP\n"
 "    -vacuum\n"
+"    -presql SQL\n"
 "\n"
 "  If the -vacuum switch is not present, argument RBU-DB must be an RBU\n"
 "  database containing an update suitable for target database TARGET-DB.\n"
@@ -76,11 +78,13 @@ int main(int argc, char **argv){
  const char *zTarget;            /* Target database to apply RBU to */
  const char *zRbu;               /* Database containing RBU */
  char zBuf[200];                 /* Buffer for printf() */
-  char *zErrmsg;                  /* Error message, if any */
+  char *zErrmsg = 0;              /* Error message, if any */
  sqlite3rbu *pRbu;               /* RBU handle */
  int nStep = 0;                  /* Maximum number of step() calls */
+  int nStatStep = 0;              /* Report stats after this many step calls */
  int bVacuum = 0;
-  int rc;
+  const char *zPreSql = 0;
+  int rc = SQLITE_OK;
  sqlite3_int64 nProgress = 0;
  int nArgc = argc-2;

@@ -90,9 +94,18 @@ int main(int argc, char **argv){
    int nArg = strlen(zArg);
    if( nArg>1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){
      bVacuum = 1;
+    }else if( nArg>1 && nArg<=7 
+           && 0==memcmp(zArg, "-presql", nArg) && i<nArg-1 ){
+      i++;
+      zPreSql = argv[i];
    }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i<nArg-1 ){
      i++;
      nStep = atoi(argv[i]);
+    }else if( nArg>1 && nArg<=9 
+           && 0==memcmp(zArg, "-statstep", nArg) && i<nArg-1 
+    ){
+      i++;
+      nStatStep = atoi(argv[i]);
    }else{
      usage(argv[0]);
    }
@@ -112,13 +125,40 @@ int main(int argc, char **argv){
  }
  report_rbu_vfs(pRbu);

+  if( zPreSql && pRbu ){
+    sqlite3 *dbMain = sqlite3rbu_db(pRbu, 0);
+    rc = sqlite3_exec(dbMain, zPreSql, 0, 0, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3 *dbRbu = sqlite3rbu_db(pRbu, 1);
+      rc = sqlite3_exec(dbRbu, zPreSql, 0, 0, 0);
+    }
+  }
+
  /* If nStep is less than or equal to zero, call
  ** sqlite3rbu_step() until either the RBU has been completely applied
  ** or an error occurs. Or, if nStep is greater than zero, call
  ** sqlite3rbu_step() a maximum of nStep times.  */
-  for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++);
-  nProgress = sqlite3rbu_progress(pRbu);
-  rc = sqlite3rbu_close(pRbu, &zErrmsg);
+  if( rc==SQLITE_OK ){
+    for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++){
+      if( nStatStep>0 && (i % nStatStep)==0 ){
+        sqlite3_int64 nUsed;
+        sqlite3_int64 nHighwater;
+        sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &nUsed, &nHighwater, 0);
+        fprintf(stdout, "memory used=%lld highwater=%lld", nUsed, nHighwater);
+        if( bVacuum==0 ){
+          int one;
+          int two;
+          sqlite3rbu_bp_progress(pRbu, &one, &two);
+          fprintf(stdout, "  progress=%d/%d\n", one, two);
+        }else{
+          fprintf(stdout, "\n");
+        }
+        fflush(stdout);
+      }
+    }
+    nProgress = sqlite3rbu_progress(pRbu);
+    rc = sqlite3rbu_close(pRbu, &zErrmsg);
+  }

  /* Let the user know what happened. */
  switch( rc ){
@@ -139,6 +139,7 @@ foreach {tn3 create_vfs destroy_vfs} {
  foreach {tn2 cmd} {
      1 run_rbu 
      2 step_rbu 3 step_rbu_uri 4 step_rbu_state
+      5 step_rbu_legacy
  } {
    foreach {tn schema} {
      1 {
@@ -669,4 +670,3 @@ foreach {tn3 create_vfs destroy_vfs} {


 finish_test
-
@@ -185,4 +185,3 @@ do_test 4.3 {


 finish_test
-
@@ -195,4 +195,3 @@ do_test 4.7.2 {
 } {1 {SQLITE_ERROR - rbu_state mismatch error}}

 finish_test
-
@@ -232,4 +232,3 @@ do_multiclient_test tn {
 }

 finish_test
-
@@ -62,4 +62,3 @@ do_execsql_test 1.4 {


 finish_test
-
@@ -92,4 +92,3 @@ foreach {tn schema} {


 finish_test
-
@@ -203,5 +203,3 @@ do_test 5.3 {
 do_test 6.1 { sqlite3rbu_internal_test } {}

 finish_test
-
-
@@ -300,7 +300,3 @@ foreach {tn idx} {


 finish_test
-
-
-
-
@@ -100,4 +100,3 @@ for {set nStep 8} {$nStep < 20} {incr nStep} {


 finish_test
-
@@ -106,5 +106,3 @@ foreach {tn tbl} {
 }

 finish_test
-
-
@@ -72,4 +72,3 @@ integrity_check 1.3.3


 finish_test
-
@@ -125,4 +125,3 @@ foreach {tn idx} {


 finish_test
-
@@ -80,4 +80,3 @@ do_test 2.2 {


 finish_test
-
@@ -59,4 +59,3 @@ sqlite3_shutdown
 test_sqlite3_log 
 sqlite3_initialize
 finish_test
-
@@ -139,4 +139,3 @@ foreach {tn schema} {


 finish_test
-
@@ -70,7 +70,24 @@ proc step_rbu {target rbu} {
  set rc
 }

+proc step_rbu_legacy {target rbu} {
+  while 1 {
+    sqlite3rbu rbu $target $rbu
+    set state [rbu state]
+    check_prestep_state $target $state
+    set rc [rbu step]
+    check_poststep_state $rc $target $state
+    rbu close
+    if {$rc != "SQLITE_OK"} break
+    sqlite3 tmpdb $rbu
+    tmpdb eval { DELETE FROM rbu_state WHERE k==10 }
+    tmpdb close
+  }
+  set rc
+}
+
 proc do_rbu_vacuum_test {tn step} {
+  forcedelete state.db
  uplevel [list do_test $tn.1 {
    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
    while 1 {
@@ -0,0 +1,62 @@
+# 2018 March 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rbucollate
+
+ifcapable !icu_collations {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+reset_db
+ 
+# Create a simple RBU database. That expects to write to a table:
+#
+#   CREATE TABLE t1(a PRIMARY KEY, b, c);
+#
+proc create_rbu1 {filename} {
+  forcedelete $filename
+  sqlite3 rbu1 $filename  
+  rbu1 eval {
+    CREATE TABLE data_t1(a, b, c, rbu_control);
+    INSERT INTO data_t1 VALUES('a', 'one', 1, 0);
+    INSERT INTO data_t1 VALUES('b', 'two', 2, 0);
+    INSERT INTO data_t1 VALUES('c', 'three', 3, 0);
+  }
+  rbu1 close
+  return $filename
+}
+
+do_execsql_test 1.0 {
+  SELECT icu_load_collation('en_US', 'my-collate');
+  CREATE TABLE t1(a COLLATE "my-collate" PRIMARY KEY, b, c);
+} {{}}
+
+do_test 1.2 {
+  create_rbu1 testrbu.db
+  sqlite3rbu rbu test.db testrbu.db
+  rbu dbMain_eval { SELECT icu_load_collation('en_US', 'my-collate') }
+  rbu dbRbu_eval { SELECT icu_load_collation('en_US', 'my-collate') }
+  while 1 {
+    set rc [rbu step]
+    if {$rc!="SQLITE_OK"} break
+  }
+  rbu close
+  db eval { SELECT * FROM t1 }
+} {a one 1 b two 2 c three 3}
+
+#forcedelete testrbu.db
+finish_test
@@ -145,4 +145,3 @@ for {set nPre 0} {$nPre < $rbu_num_steps} {incr nPre} {
 }

 finish_test
-
@@ -103,4 +103,3 @@ for {set x 1} {$x < 2} {incr x} {
 }

 finish_test
-
@@ -300,4 +300,3 @@ tablE t1 USING FTs5(c);


 finish_test
-
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .23.0
 .26.0
				`@@ -100,4 +100,3 @@ for {set nStep 8} {$nStep < 20} {incr nStep} {`


				`finish_test`