Break up the sqlite3BtreeMovetoUnpacked() routine into an eponymous routine

and sqlite3BtreeMovetoIntkey(). Each routine specializes in a single kind of btree. FossilOrigin-Name: 9a64a4f2353576ba742c2ddbf4264d00704066a1
2016-11-10 21:19:35 +00:00
1076 changed files with 32916 additions and 248565 deletions
@@ -1 +0,0 @@
-compat
@@ -1 +0,0 @@
-compat/*
@@ -1,6 +0,0 @@
-The author disclaims copyright to this source code.  In place of
-a legal notice, here is a blessing:
-
-  *   May you do good and not evil.
-  *   May you find forgiveness for yourself and forgive others.
-  *   May you share freely, never taking more than you give.
@@ -22,7 +22,7 @@ TOP = @abs_srcdir@
 #
 BCC = @BUILD_CC@ @BUILD_CFLAGS@

-# TCC is the C Compile and options for use in building executables that
+# TCC is the C Compile and options for use in building executables that 
 # will run on the target platform.  (BCC and TCC are usually the
 # same unless your are cross-compiling.)  Separate CC and CFLAGS macros
 # are provide so that these aspects of the build process can be changed
@@ -32,11 +32,10 @@ CC = @CC@
 CFLAGS = @CPPFLAGS@ @CFLAGS@
 TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
 TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
-TCC += -I${TOP}/ext/userauth

 # Define this for the autoconf-based build, so that the code knows it can
 # include the generated config.h
-#
+# 
 TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite

 # Define -DNDEBUG to compile without debugging (i.e., for production usage)
@@ -67,7 +66,7 @@ LIBREADLINE = @TARGET_READLINE_LIBS@
 TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@

 # Any target libraries which libsqlite must be linked against
-#
+# 
 TLIBS = @LIBS@ $(LIBS)

 # Flags controlling use of the in memory btree implementation
@@ -79,8 +78,8 @@ TLIBS = @LIBS@ $(LIBS)
 TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@

 # Enable/disable loadable extensions, and other optional features
-# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
-# The same set of OMIT and ENABLE flags should be passed to the
+# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).  
+# The same set of OMIT and ENABLE flags should be passed to the 
 # LEMON parser generator and the mkkeywordhash tool as well.
 OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@

@@ -90,9 +89,6 @@ TCC += $(OPT_FEATURE_FLAGS)
 # ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
 TCC += $(OPTS)

-# Add in compile-time options for some libraries used by extensions
-TCC += @HAVE_ZLIB@
-
 # Version numbers and release number for the SQLite being compiled.
 #
 VERSION = @VERSION@
@@ -127,8 +123,8 @@ SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
 # If gcov support was enabled by the configure script, add the appropriate
 # flags here.  It's not always as easy as just having the user add the right
 # CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
-# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.
-# Supposedly GCC does the right thing if you use --coverage, but in
+# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.  
+# Supposedly GCC does the right thing if you use --coverage, but in 
 # practice it still fails.  See:
 #
 # http://www.mail-archive.com/debian-gcc@lists.debian.org/msg26197.html
@@ -170,8 +166,7 @@ USE_AMALGAMATION = @USE_AMALGAMATION@
 #
 LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
         backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
-         callback.lo complete.lo ctime.lo \
-         date.lo dbpage.lo dbstat.lo delete.lo \
+         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
         expr.lo fault.lo fkey.lo \
         fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
         fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
@@ -181,18 +176,17 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
         func.lo global.lo hash.lo \
         icu.lo insert.lo json1.lo legacy.lo loadext.lo \
         main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
-         memdb.lo memjournal.lo \
+         memjournal.lo \
         mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
         notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
         pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
         random.lo resolve.lo rowset.lo rtree.lo \
-         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
+         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
         table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
-         update.lo userauth.lo upsert.lo util.lo vacuum.lo \
+         update.lo util.lo vacuum.lo \
         vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
-         vdbetrace.lo vdbevtab.lo \
-         wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
-         window.lo utf.lo vtab.lo
+         vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
+         utf.lo vtab.lo

 # Object files for the amalgamation.
 #
@@ -221,7 +215,6 @@ SRC = \
  $(TOP)/src/complete.c \
  $(TOP)/src/ctime.c \
  $(TOP)/src/date.c \
-  $(TOP)/src/dbpage.c \
  $(TOP)/src/dbstat.c \
  $(TOP)/src/delete.c \
  $(TOP)/src/expr.c \
@@ -242,7 +235,6 @@ SRC = \
  $(TOP)/src/mem2.c \
  $(TOP)/src/mem3.c \
  $(TOP)/src/mem5.c \
-  $(TOP)/src/memdb.c \
  $(TOP)/src/memjournal.c \
  $(TOP)/src/msvc.h \
  $(TOP)/src/mutex.c \
@@ -273,7 +265,7 @@ SRC = \
  $(TOP)/src/rowset.c \
  $(TOP)/src/select.c \
  $(TOP)/src/status.c \
-  $(TOP)/src/shell.c.in \
+  $(TOP)/src/shell.c \
  $(TOP)/src/sqlite.h.in \
  $(TOP)/src/sqlite3ext.h \
  $(TOP)/src/sqliteInt.h \
@@ -286,7 +278,6 @@ SRC = \
  $(TOP)/src/trigger.c \
  $(TOP)/src/utf.c \
  $(TOP)/src/update.c \
-  $(TOP)/src/upsert.c \
  $(TOP)/src/util.c \
  $(TOP)/src/vacuum.c \
  $(TOP)/src/vdbe.c \
@@ -297,7 +288,6 @@ SRC = \
  $(TOP)/src/vdbemem.c \
  $(TOP)/src/vdbesort.c \
  $(TOP)/src/vdbetrace.c \
-  $(TOP)/src/vdbevtab.c \
  $(TOP)/src/vdbeInt.h \
  $(TOP)/src/vtab.c \
  $(TOP)/src/vxworks.h \
@@ -307,8 +297,7 @@ SRC = \
  $(TOP)/src/where.c \
  $(TOP)/src/wherecode.c \
  $(TOP)/src/whereexpr.c \
-  $(TOP)/src/whereInt.h \
-  $(TOP)/src/window.c
+  $(TOP)/src/whereInt.h

 # Source code for extensions
 #
@@ -353,20 +342,15 @@ SRC += \
  $(TOP)/ext/icu/icu.c
 SRC += \
  $(TOP)/ext/rtree/rtree.h \
-  $(TOP)/ext/rtree/rtree.c \
-  $(TOP)/ext/rtree/geopoly.c
+  $(TOP)/ext/rtree/rtree.c
 SRC += \
  $(TOP)/ext/session/sqlite3session.c \
  $(TOP)/ext/session/sqlite3session.h
-SRC += \
-  $(TOP)/ext/userauth/userauth.c \
-  $(TOP)/ext/userauth/sqlite3userauth.h
 SRC += \
  $(TOP)/ext/rbu/sqlite3rbu.h \
  $(TOP)/ext/rbu/sqlite3rbu.c
 SRC += \
-  $(TOP)/ext/misc/json1.c \
-  $(TOP)/ext/misc/stmt.c
+  $(TOP)/ext/misc/json1.c

 # Generated source code files
 #
@@ -377,7 +361,6 @@ SRC += \
  parse.c \
  parse.h \
  config.h \
-  shell.c \
  sqlite3.h

 # Source code to the test files.
@@ -409,7 +392,6 @@ TESTSRC = \
  $(TOP)/src/test_intarray.c \
  $(TOP)/src/test_journal.c \
  $(TOP)/src/test_malloc.c \
-  $(TOP)/src/test_md5.c \
  $(TOP)/src/test_multiplex.c \
  $(TOP)/src/test_mutex.c \
  $(TOP)/src/test_onefile.c \
@@ -421,50 +403,37 @@ TESTSRC = \
  $(TOP)/src/test_server.c \
  $(TOP)/src/test_superlock.c \
  $(TOP)/src/test_syscall.c \
-  $(TOP)/src/test_tclsh.c \
  $(TOP)/src/test_tclvar.c \
  $(TOP)/src/test_thread.c \
-  $(TOP)/src/test_vdbecov.c \
  $(TOP)/src/test_vfs.c \
  $(TOP)/src/test_windirent.c \
-  $(TOP)/src/test_window.c \
  $(TOP)/src/test_wsd.c       \
  $(TOP)/ext/fts3/fts3_term.c \
  $(TOP)/ext/fts3/fts3_test.c  \
  $(TOP)/ext/session/test_session.c \
-  $(TOP)/ext/rbu/test_rbu.c
+  $(TOP)/ext/rbu/test_rbu.c 

 # Statically linked extensions
 #
 TESTSRC += \
-  $(TOP)/ext/expert/sqlite3expert.c \
-  $(TOP)/ext/expert/test_expert.c \
  $(TOP)/ext/misc/amatch.c \
  $(TOP)/ext/misc/carray.c \
  $(TOP)/ext/misc/closure.c \
  $(TOP)/ext/misc/csv.c \
  $(TOP)/ext/misc/eval.c \
-  $(TOP)/ext/misc/explain.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
  $(TOP)/ext/fts5/fts5_test_mi.c \
  $(TOP)/ext/fts5/fts5_test_tok.c \
  $(TOP)/ext/misc/ieee754.c \
-  $(TOP)/ext/misc/mmapwarm.c \
  $(TOP)/ext/misc/nextchar.c \
-  $(TOP)/ext/misc/normalize.c \
  $(TOP)/ext/misc/percentile.c \
-  $(TOP)/ext/misc/prefixes.c \
  $(TOP)/ext/misc/regexp.c \
-  $(TOP)/ext/misc/remember.c \
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \
-  $(TOP)/ext/misc/unionvtab.c \
-  $(TOP)/ext/misc/wholenumber.c \
-  $(TOP)/ext/misc/zipfile.c \
-  $(TOP)/ext/userauth/userauth.c
+  $(TOP)/ext/misc/wholenumber.c

 # Source code to the library files needed by the test fixture
 #
@@ -476,11 +445,9 @@ TESTSRC2 = \
  $(TOP)/src/build.c \
  $(TOP)/src/ctime.c \
  $(TOP)/src/date.c \
-  $(TOP)/src/dbpage.c \
  $(TOP)/src/dbstat.c \
  $(TOP)/src/expr.c \
  $(TOP)/src/func.c \
-  $(TOP)/src/global.c \
  $(TOP)/src/insert.c \
  $(TOP)/src/wal.c \
  $(TOP)/src/main.c \
@@ -504,11 +471,9 @@ TESTSRC2 = \
  $(TOP)/src/vdbe.c \
  $(TOP)/src/vdbemem.c \
  $(TOP)/src/vdbetrace.c \
-  $(TOP)/src/vdbevtab.c \
  $(TOP)/src/where.c \
  $(TOP)/src/wherecode.c \
  $(TOP)/src/whereexpr.c \
-  $(TOP)/src/window.c \
  parse.c \
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3_aux.c \
@@ -517,8 +482,7 @@ TESTSRC2 = \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_write.c \
  $(TOP)/ext/async/sqlite3async.c \
-  $(TOP)/ext/session/sqlite3session.c \
-  $(TOP)/ext/misc/stmt.c
+  $(TOP)/ext/session/sqlite3session.c 

 # Header files used by all library source files.
 #
@@ -565,14 +529,11 @@ EXTHDR += \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_tokenizer.h
 EXTHDR += \
-  $(TOP)/ext/rtree/rtree.h \
-  $(TOP)/ext/rtree/geopoly.c
+  $(TOP)/ext/rtree/rtree.h
 EXTHDR += \
  $(TOP)/ext/icu/sqliteicu.h
 EXTHDR += \
  $(TOP)/ext/rtree/sqlite3rtree.h
-EXTHDR += \
-  $(TOP)/ext/userauth/sqlite3userauth.h

 # executables needed for testing
 #
@@ -581,8 +542,7 @@ TESTPROGS = \
  sqlite3$(TEXE) \
  sqlite3_analyzer$(TEXE) \
  sqldiff$(TEXE) \
-  dbhash$(TEXE) \
-  sqltclsh$(TEXE)
+  dbhash$(TEXE)

 # Databases containing fuzzer test cases
 #
@@ -590,11 +550,7 @@ FUZZDATA = \
  $(TOP)/test/fuzzdata1.db \
  $(TOP)/test/fuzzdata2.db \
  $(TOP)/test/fuzzdata3.db \
-  $(TOP)/test/fuzzdata4.db \
-  $(TOP)/test/fuzzdata5.db \
-  $(TOP)/test/fuzzdata6.db \
-  $(TOP)/test/fuzzdata7.db \
-  $(TOP)/test/fuzzdata8.db
+  $(TOP)/test/fuzzdata4.db

 # Standard options to testfixture
 #
@@ -603,29 +559,11 @@ TESTOPTS = --verbose=file --output=test-out.txt
 # Extra compiler options for various shell tools
 #
 SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
-#SHELL_OPT += -DSQLITE_ENABLE_FTS5
-SHELL_OPT += -DSQLITE_ENABLE_RTREE
+# SHELL_OPT += -DSQLITE_ENABLE_FTS5
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
-SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
-SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
-SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
-SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
-SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
-SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
-FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
-FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
-FUZZCHECK_OPT += -DSQLITE_ENABLE_DESERIALIZE
-FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4
-#FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS5
-FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE
-FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY
-FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
-FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
-FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
-DBFUZZ_OPT =
+FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5

 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
@@ -649,9 +587,9 @@ libtclsqlite3.la:	tclsqlite.lo libsqlite3.la
 		-version-info "8:6:8" \
 		-avoid-version

-sqlite3$(TEXE):	shell.c sqlite3.c
+sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
 	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
-		shell.c sqlite3.c \
+		$(TOP)/src/shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"

 sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
@@ -674,50 +612,8 @@ fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
 	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)

-fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h
-	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
-
-ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
-	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
-             $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
-
-sessionfuzz$(TEXE):	$(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h
-	$(LTLINK) -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)
-
-dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
-	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
-
-DBFUZZ2_OPTS = \
-  -DSQLITE_THREADSAFE=0 \
-  -DSQLITE_OMIT_LOAD_EXTENSION \
-  -DSQLITE_ENABLE_DESERIALIZE \
-  -DSQLITE_DEBUG \
-  -DSQLITE_ENABLE_DBSTAT_VTAB \
-  -DSQLITE_ENABLE_BYTECODE_VTAB \
-  -DSQLITE_ENABLE_RTREE \
-  -DSQLITE_ENABLE_FTS4 \
-  -DSQLITE_ENABLE_FTS5
-
-dbfuzz2$(TEXE):	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
-	$(CC) $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \
-		-DSTANDALONE -o dbfuzz2 \
-		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(TLIBS)
-	mkdir -p dbfuzz2-dir
-	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
-
-dbfuzz2-asan:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
-	clang-6.0 $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \
-		-fsanitize=fuzzer,undefined,address -o dbfuzz2-asan \
-		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(TLIBS)
-	mkdir -p dbfuzz2-dir
-	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
-
-dbfuzz2-msan:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
-	clang-6.0 $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \
-		-fsanitize=fuzzer,undefined,memory -o dbfuzz2-msan \
-		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(TLIBS)
-	mkdir -p dbfuzz2-dir
-	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
+fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
+	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)

 mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
 	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
@@ -755,7 +651,7 @@ mptest:	mptester$(TEXE)

 sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
-	cp tsrc/sqlite3ext.h .
+	cp tsrc/shell.c tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .

 sqlite3ext.h:	.target_source
@@ -781,11 +677,6 @@ lemon$(BEXE):	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
 	$(BCC) -o $@ $(TOP)/tool/lemon.c
 	cp $(TOP)/tool/lempar.c .

-# Rules to build the program that generates the source-id
-#
-mksourceid$(BEXE):	$(TOP)/tool/mksourceid.c
-	$(BCC) -o $@ $(TOP)/tool/mksourceid.c
-
 # Rules to build individual *.o files from generated *.c files. This
 # applies to:
 #
@@ -839,9 +730,6 @@ ctime.lo:	$(TOP)/src/ctime.c $(HDR)
 date.lo:	$(TOP)/src/date.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c

-dbpage.lo:	$(TOP)/src/dbpage.c $(HDR)
-	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbpage.c
-
 dbstat.lo:	$(TOP)/src/dbstat.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c

@@ -896,9 +784,6 @@ mem3.lo:	$(TOP)/src/mem3.c $(HDR)
 mem5.lo:	$(TOP)/src/mem5.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem5.c

-memdb.lo:	$(TOP)/src/memdb.c $(HDR)
-	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memdb.c
-
 memjournal.lo:	$(TOP)/src/memjournal.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memjournal.c

@@ -977,9 +862,6 @@ trigger.lo:	$(TOP)/src/trigger.c $(HDR)
 update.lo:	$(TOP)/src/update.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/update.c

-upsert.lo:	$(TOP)/src/upsert.c $(HDR)
-	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/upsert.c
-
 utf.lo:	$(TOP)/src/utf.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/utf.c

@@ -1010,9 +892,6 @@ vdbesort.lo:	$(TOP)/src/vdbesort.c $(HDR)
 vdbetrace.lo:	$(TOP)/src/vdbetrace.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbetrace.c

-vdbevtab.lo:	$(TOP)/src/vdbevtab.c $(HDR)
-	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbevtab.c
-
 vtab.lo:	$(TOP)/src/vtab.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vtab.c

@@ -1031,14 +910,11 @@ wherecode.lo:	$(TOP)/src/wherecode.c $(HDR)
 whereexpr.lo:	$(TOP)/src/whereexpr.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/whereexpr.c

-window.lo:	$(TOP)/src/window.c $(HDR)
-	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/window.c
-
 tclsqlite.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c

 tclsqlite-shell.lo:	$(TOP)/src/tclsqlite.c $(HDR)
-	$(LTCOMPILE) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c
+	$(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c

 tclsqlite-stubs.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
@@ -1059,36 +935,20 @@ opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl
 #
 parse.h:	parse.c

-parse.c:	$(TOP)/src/parse.y lemon$(BEXE)
+parse.c:	$(TOP)/src/parse.y lemon$(BEXE) $(TOP)/tool/addopcodes.tcl
 	cp $(TOP)/src/parse.y .
-	./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) -S parse.y
+	rm -f parse.h
+	./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
+	mv parse.h parse.h.temp
+	$(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h

-sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
+sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h

 keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
 	./mkkeywordhash$(BEXE) >keywordhash.h

-# Source files that go into making shell.c
-SHELL_SRC = \
-	$(TOP)/src/shell.c.in \
-        $(TOP)/ext/misc/appendvfs.c \
-	$(TOP)/ext/misc/shathree.c \
-	$(TOP)/ext/misc/fileio.c \
-	$(TOP)/ext/misc/completion.c \
-	$(TOP)/ext/misc/sqlar.c \
-        $(TOP)/ext/misc/uint.c \
-	$(TOP)/ext/expert/sqlite3expert.c \
-	$(TOP)/ext/expert/sqlite3expert.h \
-	$(TOP)/ext/misc/zipfile.c \
-	$(TOP)/ext/misc/memtrace.c \
-        $(TOP)/src/test_windirent.c
-
-shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
-	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
-
-


 # Rules to build the extension objects.
@@ -1156,18 +1016,12 @@ fts3_write.lo:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
 rtree.lo:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c

-userauth.lo:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
-	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c
-
 sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c

 json1.lo:	$(TOP)/ext/misc/json1.c
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c

-stmt.lo:	$(TOP)/ext/misc/stmt.c
-	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c
-
 # FTS5 things
 #
 FTS5_SRC = \
@@ -1187,10 +1041,10 @@ FTS5_SRC = \
   $(TOP)/ext/fts5/fts5_varint.c \
   $(TOP)/ext/fts5/fts5_vocab.c  \

-fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon$(BEXE)
+fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
 	cp $(TOP)/ext/fts5/fts5parse.y .
 	rm -f fts5parse.h
-	./lemon$(BEXE) $(OPTS) -S fts5parse.y
+	./lemon$(BEXE) $(OPTS) fts5parse.y

 fts5parse.h: fts5parse.c

@@ -1212,16 +1066,11 @@ sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 # necessary because the test fixture requires non-API symbols which are
 # hidden when the library is built via the amalgamation).
 #
-TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
-TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
-TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
+TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
-TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
-TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
-TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_BYTECODE_VTAB
-TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DESERIALIZE

 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
 TESTFIXTURE_SRC1 = sqlite3.c
@@ -1232,10 +1081,6 @@ testfixture$(TEXE):	$(TESTFIXTURE_SRC)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

-coretestprogs:	$(TESTPROGS)
-
-testprogs:	coretestprogs srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE)
-
 # A very detailed test running most or all test cases
 fulltest:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/all.test $(TESTOPTS)
@@ -1249,18 +1094,14 @@ fulltestonly:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/full.test

 # Fuzz testing
-fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
+fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
 	./fuzzcheck$(TEXE) $(FUZZDATA)
-	./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db

-valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
+fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
+	./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA)
+
+valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
 	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)
-	valgrind ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
-
-# The veryquick.test TCL tests.
-#
-tcltest:	./testfixture$(TEXE)
-	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)

 # Minimal testing that runs in less than 3 minutes
 #
@@ -1270,7 +1111,8 @@ quicktest:	./testfixture$(TEXE)
 # This is the common case.  Run many tests that do not take too long,
 # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
 #
-test:	fuzztest sourcetest $(TESTPROGS) tcltest
+test:	$(TESTPROGS) sourcetest fastfuzztest
+	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)

 # Run a test using valgrind.  This can take a really long time
 # because valgrind is so much slower than a native machine.
@@ -1285,44 +1127,18 @@ valgrindtest:	$(TESTPROGS) valgrindfuzz
 smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)

-sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
-	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
+sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
+	echo "#define TCLSH 2" > $@
+	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
+	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
+	echo "static const char *tclsh_main_loop(void){" >> $@
+	echo "static const char *zMainloop = " >> $@
+	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
+	echo "; return zMainloop; }" >> $@

 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)

-sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
-	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
-
-sqltclsh$(TEXE): sqltclsh.c
-	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
-
-sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
-	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
-
-CHECKER_DEPS =\
-  $(TOP)/tool/mkccode.tcl \
-  sqlite3.c \
-  $(TOP)/src/tclsqlite.c \
-  $(TOP)/ext/repair/sqlite3_checker.tcl \
-  $(TOP)/ext/repair/checkindex.c \
-  $(TOP)/ext/repair/checkfreelist.c \
-  $(TOP)/ext/misc/btreeinfo.c \
-  $(TOP)/ext/repair/sqlite3_checker.c.in
-
-sqlite3_checker.c:	$(CHECKER_DEPS)
-	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
-
-sqlite3_checker$(TEXE):	sqlite3_checker.c
-	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
-
-dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
-	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
-           $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
-
-dbtotxt$(TEXE): $(TOP)/tool/dbtotxt.c
-	$(LTLINK)-o $@ $(TOP)/tool/dbtotxt.c
-
 showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)

@@ -1335,39 +1151,22 @@ showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
 showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)

-showshm$(TEXE):	$(TOP)/tool/showshm.c
-	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
-
-index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
-	$(LTLINK) $(SHELL_OPT) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)
-
 changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)

-changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
-	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)
-
 rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)

-atrc$(TEXX): $(TOP)/test/atrc.c sqlite3.lo
-	$(LTLINK) -o $@ $(TOP)/test/atrc.c sqlite3.lo $(TLIBS)
-
 LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c

 wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)

-speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.c
-	$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)
+speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)

-KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
-
-kvtest$(TEXE):	$(TOP)/test/kvtest.c sqlite3.c
-	$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)
-
-rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo
+rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
 	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)

 loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
@@ -1378,9 +1177,8 @@ loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
 # releasetest.tcl script.
 #
 VALIDIDS=' sqlite3(changeset|changegroup|session)?_'
-checksymbols: sqlite3.o
-	nm -g --defined-only sqlite3.o
-	nm -g --defined-only sqlite3.o | egrep -v $(VALIDIDS); test $$? -ne 0
+checksymbols: sqlite3.lo
+	nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0
 	echo '0 errors out of 1 tests'

 # Build the amalgamation-autoconf package.  The amalamgation-tarball target builds
@@ -1396,7 +1194,7 @@ snapshot-tarball: sqlite3.c
 # The next two rules are used to support the "threadtest" target. Building
 # threadtest runs a few thread-safety tests that are implemented in C. This
 # target is invoked by the releasetest.tcl script.
-#
+# 
 THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
                  $(TOP)/test/tt3_checkpoint.c \
                  $(TOP)/test/tt3_index.c      \
@@ -1410,7 +1208,7 @@ threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC)
 threadtest: threadtest3$(TEXE)
 	./threadtest3$(TEXE)

-releasetest:
+releasetest:	
 	$(TCLSH_CMD) $(TOP)/test/releasetest.tcl

 # Standard install and cleanup targets
@@ -1418,7 +1216,7 @@ releasetest:
 lib_install:	libsqlite3.la
 	$(INSTALL) -d $(DESTDIR)$(libdir)
 	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
-
+	
 install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
 	$(INSTALL) -d $(DESTDIR)$(bindir)
 	$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
@@ -1429,14 +1227,14 @@ install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_instal
 	$(INSTALL) -m 0644 sqlite3.pc $(DESTDIR)$(pkgconfigdir)

 pkgIndex.tcl:
-	echo 'package ifneeded sqlite3 $(RELEASE) [list load [file join $$dir libtclsqlite3[info sharedlibextension]] sqlite3]' > $@
+	echo 'package ifneeded sqlite3 $(RELEASE) [list load $(TCLLIBDIR)/libtclsqlite3$(SHLIB_SUFFIX) sqlite3]' > $@
 tcl_install:	lib_install libtclsqlite3.la pkgIndex.tcl
 	$(INSTALL) -d $(DESTDIR)$(TCLLIBDIR)
 	$(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)
 	rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a
 	$(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)

-clean:
+clean:	
 	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
 	rm -f sqlite3.h opcodes.*
 	rm -rf .libs .deps
@@ -19,7 +19,7 @@ TOP = ../sqlite
 #### C Compiler and options for use in building executables that
 #    will run on the platform that is doing the build.
 #
-BCC = gcc -g -O0
+BCC = gcc -g -O2
 #BCC = /opt/ancic/bin/c89 -0

 #### If the target operating system supports the "usleep()" system
@@ -38,8 +38,8 @@ THREADSAFE = -DTHREADSAFE=0
 #### Specify any extra linker options needed to make the library
 #    thread safe
 #
-THREADLIB = -lpthread -lm -ldl
-#THREADLIB = 
+#THREADLIB = -lpthread
+THREADLIB = 

 #### Specify any extra libraries needed to access required functions.
 #
@@ -54,9 +54,11 @@ TLIBS =
 #    You can make the library go almost twice as fast if you compile
 #    with -DNDEBUG=1
 #
-OPTS += -DSQLITE_DEBUG=1
-OPTS += -DSQLITE_ENABLE_WHERETRACE
-OPTS += -DSQLITE_ENABLE_SELECTTRACE
+#OPTS = -DSQLITE_DEBUG=2
+#OPTS = -DSQLITE_DEBUG=1
+#OPTS = 
+OPTS = -DNDEBUG=1
+OPTS += -DHAVE_FDATASYNC=1

 #### The suffix to add to executable files.  ".exe" for windows.
 #    Nothing for unix.
@@ -68,7 +70,7 @@ EXE =
 #    will run on the target platform.  This is usually the same
 #    as BCC, unless you are cross-compiling.
 #
-TCC = gcc -O0
+TCC = gcc -O6
 #TCC = gcc -g -O0 -Wall
 #TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage
 #TCC = /opt/mingw/bin/i386-mingw32-gcc -O6
@@ -89,12 +91,18 @@ SHPREFIX = lib

 #### Extra compiler options needed for programs that use the TCL library.
 #
-TCL_FLAGS = -I/home/drh/tcl/include/tcl8.6
+#TCL_FLAGS =
+#TCL_FLAGS = -DSTATIC_BUILD=1
+TCL_FLAGS = -I/home/drh/tcltk/8.5linux
+#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1
+#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux

 #### Linker options needed to link against the TCL library.
 #
 #LIBTCL = -ltcl -lm -ldl
-LIBTCL = /home/drh/tcl/lib/libtcl8.6.a -lm -lpthread -ldl -lz
+LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl
+#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt
+#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc

 #### Additional objects for SQLite library when TCL support is enabled.
 #TCLOBJ =
@@ -1,72 +1,12 @@
 <h1 align="center">SQLite Source Repository</h1>

-This repository contains the complete source code for the 
-[SQLite database engine](https://sqlite.org/).  Some test scripts 
-are also included.  However, many other test scripts
+This repository contains the complete source code for the SQLite database
+engine.  Some test scripts are also include.  However, many other test scripts
 and most of the documentation are managed separately.

-## Version Control
-
-SQLite sources are managed using the
-[Fossil](https://www.fossil-scm.org/), a distributed version control system
-that was specifically designed and written to support SQLite development.
-The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext.
-
-If you are reading this on GitHub or some other Git repository or service,
-then you are looking at a mirror.  The names of check-ins and
-other artifacts in a Git mirror are different from the official
-names for those objects.  The offical names for check-ins are
-found in a footer on the check-in comment for authorized mirrors.
-The official check-in name can also be seen in the `manifest.uuid` file
-in the root of the tree.  Always use the official name, not  the
-Git-name, when communicating about an SQLite check-in.
-
-If you pulled your SQLite source code from a secondary source and want to
-verify its integrity, there are hints on how to do that in the
-[Verifying Code Authenticity](#vauth) section below.
-
-## Obtaining The Code
-
-If you do not want to use Fossil, you can download tarballs or ZIP
-archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
-
-  *  Lastest trunk check-in as
-     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz),
-     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or
-     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar).
-
-  *  Latest release as
-     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release),
-     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or
-     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release).
-
-  *  For other check-ins, substitute an appropriate branch name or
-     tag or hash prefix in place of "release" in the URLs of the previous
-     bullet.  Or browse the [timeline](https://www.sqlite.org/src/timeline)
-     to locate the check-in desired, click on its information page link,
-     then click on the "Tarball" or "ZIP Archive" links on the information
-     page.
-
-If you do want to use Fossil to check out the source tree, 
-first install Fossil version 2.0 or later.
-(Source tarballs and precompiled binaries available
-[here](https://www.fossil-scm.org/fossil/uv/download.html).  Fossil is
-a stand-alone program.  To install, simply download or build the single 
-executable file and put that file someplace on your $PATH.)
-Then run commands like this:
-
-        mkdir ~/sqlite
-        cd ~/sqlite
-        fossil clone https://www.sqlite.org/src sqlite.fossil
-        fossil open sqlite.fossil
-    
-After setting up a repository using the steps above, you can always
-update to the lastest version using:
-
-        fossil update trunk   ;# latest trunk check-in
-        fossil update release ;# latest official release
-
-Or type "fossil ui" to get a web-based user interface.
+If you are reading this on a Git mirror someplace, you are doing it wrong.
+The [official repository](https://www.sqlite.org/src/) is better.  Go there
+now.

 ## Compiling

@@ -78,13 +18,13 @@ script found at the root of the source tree.  Then run "make".

 For example:

-        tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
-        mkdir bld                ;#  Build will occur in a sibling directory
-        cd bld                   ;#  Change to the build directory
-        ../sqlite/configure      ;#  Run the configure script
-        make                     ;#  Run the makefile.
-        make sqlite3.c           ;#  Build the "amalgamation" source file
-        make test                ;#  Run some tests (requires Tcl)
+    tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
+    mkdir bld                ;#  Build will occur in a sibling directory
+    cd bld                   ;#  Change to the build directory
+    ../sqlite/configure      ;#  Run the configure script
+    make                     ;#  Run the makefile.
+    make sqlite3.c           ;#  Build the "amalgamation" source file
+    make test                ;#  Run some tests (requires Tcl)

 See the makefile for additional targets.

@@ -103,13 +43,13 @@ with the provided "Makefile.msc" to build one of the supported targets.

 For example:

-        mkdir bld
-        cd bld
-        nmake /f Makefile.msc TOP=..\sqlite
-        nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
-        nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
-        nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
-        nmake /f Makefile.msc test TOP=..\sqlite
+    mkdir bld
+    cd bld
+    nmake /f Makefile.msc TOP=..\sqlite
+    nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
+    nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
+    nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
+    nmake /f Makefile.msc test TOP=..\sqlite

 There are several build options that can be set via the NMAKE command
 line.  For example, to build for WinRT, simply add "FOR_WINRT=1" argument
@@ -120,25 +60,23 @@ recommended.
 SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
 is required by the makefiles (including those for MSVC).  SQLite contains
 a lot of generated code and Tcl is used to do much of that code generation.
+The makefiles also require AWK.

 ## Source Code Tour

-Most of the core source files are in the **src/** subdirectory.  The
-**src/** folder also contains files used to build the "testfixture" test
-harness. The names of the source files used by "testfixture" all begin
-with "test".
-The **src/** also contains the "shell.c" file
-which is the main program for the "sqlite3.exe"
-[command-line shell](https://sqlite.org/cli.html) and
-the "tclsqlite.c" file which implements the
-[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
-(Historical note:  SQLite began as a Tcl
+Most of the core source files are in the **src/** subdirectory.  But
+src/ also contains files used to build the "testfixture" test harness;
+those file all begin with "test".  And src/ contains the "shell.c" file
+which is the main program for the "sqlite3.exe" command-line shell and
+the "tclsqlite.c" file which implements the bindings to SQLite from the
+Tcl programming language.  (Historical note:  SQLite began as a Tcl
 extension and only later escaped to the wild as an independent library.)

 Test scripts and programs are found in the **test/** subdirectory.
-Addtional test code is found in other source repositories.
-See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
-additional information.
+There are other test suites for SQLite (see
+[How SQLite Is Tested](http://www.sqlite.org/testing.html))
+but those other test suites are
+in separate source repositories.

 The **ext/** subdirectory contains code for extensions.  The
 Full-text search engine is in **ext/fts3**.  The R-Tree engine is in
@@ -162,7 +100,7 @@ manually-edited files and automatically-generated files.
 The SQLite interface is defined by the **sqlite3.h** header file, which is
 generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
 [Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
-The manifest.uuid file contains the SHA3 hash of the particular check-in
+The manifest.uuid file contains the SHA1 hash of the particular check-in
 and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
 contains the current SQLite version number.  The sqlite3.h header is really
 just a copy of src/sqlite.h.in with the source-id and version number inserted
@@ -173,16 +111,20 @@ used to generate that documentation are in a separate source repository.
 The SQL language parser is **parse.c** which is generate from a grammar in
 the src/parse.y file.  The conversion of "parse.y" into "parse.c" is done
 by the [lemon](./doc/lemon.html) LALR(1) parser generator.  The source code
-for lemon is at tool/lemon.c.  Lemon uses the tool/lempar.c file as a
-template for generating its parser.
+for lemon is at tool/lemon.c.  Lemon uses a
+template for generating its parser.  A generic template is in tool/lempar.c,
+but SQLite uses a slightly modified template found in src/lempar.c.
+
 Lemon also generates the **parse.h** header file, at the same time it
-generates parse.c.
+generates parse.c. But the parse.h header file is
+modified further (to add additional symbols) using the ./addopcodes.awk
+AWK script.

 The **opcodes.h** header file contains macros that define the numbers
 corresponding to opcodes in the "VDBE" virtual machine.  The opcodes.h
 file is generated by the scanning the src/vdbe.c source file.  The
-Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h.
-A second Tcl script, ./mkopcodec.tcl, then scans opcodes.h to generate
+AWK script at ./mkopcodeh.awk does this scan and generates opcodes.h.
+A second AWK script, ./mkopcodec.awk, then scans opcodes.h to generate
 the **opcodes.c** source file, which contains a reverse mapping from
 opcode-number to opcode-name that is used for EXPLAIN output.

@@ -191,13 +133,6 @@ that maps SQL language keywords (ex: "CREATE", "SELECT", "INDEX", etc.) into
 the numeric codes used by the parse.c parser.  The keywordhash.h file is
 generated by a C-language program at tool mkkeywordhash.c.

-The **pragma.h** header file contains various definitions used to parse
-and implement the PRAGMA statements.  The header is generated by a
-script **tool/mkpragmatab.tcl**. If you want to add a new PRAGMA, edit
-the **tool/mkpragmatab.tcl** file to insert the information needed by the
-parser for your new PRAGMA, then run the script to regenerate the
-**pragma.h** header file.
-
 ### The Amalgamation

 All of the individual C source code and header files (both manually-edited
@@ -215,12 +150,12 @@ subdirectory (using the equivalent of "make target_source") then the
 tool/mksqlite3c.tcl script is run to copy them all together in just the
 right order while resolving internal "#include" references.

-The amalgamation source file is more than 200K lines long.  Some symbolic
+The amalgamation source file is more than 100K lines long.  Some symbolic
 debuggers (most notably MSVC) are unable to deal with files longer than 64K
 lines.  To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
 can be run on the amalgamation to break it up into a single small C file
-called **sqlite3-all.c** that does #include on about seven other files
-named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-7.c**.  In this way,
+called **sqlite3-all.c** that does #include on about five other files
+named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**.  In this way,
 all of the source code is contained within a single translation unit so
 that the compiler can do extra cross-procedure optimization, but no
 individual source file exceeds 32K lines in length.
@@ -232,15 +167,14 @@ See the [architectural description](http://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
 (helping to understand how SQLite works include the
 [file format](http://www.sqlite.org/fileformat2.html) description,
-the [virtual machine](http://www.sqlite.org/opcode.html) that runs
+the [virtual machine](http://www.sqlite.org/vdbe.html) that runs
 prepared statements, the description of
 [how transactions work](http://www.sqlite.org/atomiccommit.html), and
 the [overview of the query planner](http://www.sqlite.org/optoverview.html).

-Years of effort have gone into optimizating SQLite, both
+Unfortunately, years of effort have gone into optimizating SQLite, both
 for small size and high performance.  And optimizations tend to result in
-complex code.  So there is a lot of complexity in the current SQLite
-implementation.  It will not be the easiest library in the world to hack.
+complex code.  So there is a lot of complexity in the SQLite implementation.

 Key files:

@@ -249,10 +183,9 @@ Key files:
     trying to understand how the library works internally.

  *  **sqliteInt.h** - this header file defines many of the data objects
-     used internally by SQLite.  In addition to "sqliteInt.h", some
-     subsystems have their own header files.
+     used internally by SQLite.

-  *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
+  *  **parse.y** - This file describes the LALR(1) grammer that SQLite uses
     to parse SQL statements, and the actions that are taken at each step
     in the parsing process.

@@ -262,66 +195,36 @@ Key files:
     which defines internal data objects.  The rest of SQLite interacts
     with the VDBE through an interface defined by vdbe.h.

-  *  **where.c** - This file (together with its helper files named
-     by "where*.c") analyzes the WHERE clause and generates
+  *  **where.c** - This file analyzes the WHERE clause and generates
     virtual machine code to run queries efficiently.  This file is
     sometimes called the "query optimizer".  It has its own private
     header file, whereInt.h, that defines data objects used internally.

  *  **btree.c** - This file contains the implementation of the B-Tree
-     storage engine used by SQLite.  The interface to the rest of the system
-     is defined by "btree.h".  The "btreeInt.h" header defines objects
-     used internally by btree.c and not published to the rest of the system.
+     storage engine used by SQLite.

  *  **pager.c** - This file contains the "pager" implementation, the
-     module that implements transactions.  The "pager.h" header file
-     defines the interface between pager.c and the rest of the system.
+     module that implements transactions.

  *  **os_unix.c** and **os_win.c** - These two files implement the interface
     between SQLite and the underlying operating system using the run-time
     pluggable VFS interface.

-  *  **shell.c.in** - This file is not part of the core SQLite library.  This
+  *  **shell.c** - This file is not part of the core SQLite library.  This
     is the file that, when linked against sqlite3.a, generates the
-     "sqlite3.exe" command-line shell.  The "shell.c.in" file is transformed
-     into "shell.c" as part of the build process.
+     "sqlite3.exe" command-line shell.

  *  **tclsqlite.c** - This file implements the Tcl bindings for SQLite.  It
     is not part of the core SQLite library.  But as most of the tests in this
     repository are written in Tcl, the Tcl language bindings are important.

-  *  **test*.c** - Files in the src/ folder that begin with "test" go into
-     building the "testfixture.exe" program.  The testfixture.exe program is
-     an enhanced Tcl shell.  The testfixture.exe program runs scripts in the
-     test/ folder to validate the core SQLite code.  The testfixture program
-     (and some other test programs too) is build and run when you type
-     "make test".
-
-  *  **ext/misc/json1.c** - This file implements the various JSON functions
-     that are build into SQLite.
-
-There are many other source files.  Each has a succinct header comment that
+There are many other source files.  Each has a suscinct header comment that
 describes its purpose and role within the larger system.

-<a name="vauth"></a>
-## Verifying Code Authenticity
-
-The `manifest` file at the root directory of the source tree
-contains either a SHA3-256 hash (for newer files) or a SHA1 hash (for 
-older files) for every source file in the repository.
-The SHA3-256 hash of the `manifest`
-file itself is the official name of the version of the source tree that you
-have. The `manifest.uuid` file should contain the SHA3-256 hash of the
-`manifest` file. If all of the above hash comparisons are correct, then
-you can be confident that your source tree is authentic and unadulterated.
-
-The format of the `manifest` file should be mostly self-explanatory, but
-if you want details, they are available
-[here](https://fossil-scm.org/fossil/doc/trunk/www/fileformat.wiki#manifest).

 ## Contacts

-The main SQLite website is [http://www.sqlite.org/](http://www.sqlite.org/)
-with geographically distributed backups at
+The main SQLite webpage is [http://www.sqlite.org/](http://www.sqlite.org/)
+with geographically distributed backup servers at
 [http://www2.sqlite.org/](http://www2.sqlite.org) and
 [http://www3.sqlite.org/](http://www3.sqlite.org).
@@ -1 +1 @@
-3.32.0
+3.16.0
@@ -1,5 +1,6 @@

-AM_CFLAGS = @BUILD_CFLAGS@ 
+AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
+
 lib_LTLIBRARIES = libsqlite3.la
 libsqlite3_la_SOURCES = sqlite3.c
 libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
@@ -9,11 +10,11 @@ sqlite3_SOURCES = shell.c sqlite3.h
 EXTRA_sqlite3_SOURCES = sqlite3.c
 sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
 sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
-sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
+sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS

 include_HEADERS = sqlite3.h sqlite3ext.h

-EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs Makefile.fallback
+EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
 pkgconfigdir = ${libdir}/pkgconfig
 pkgconfig_DATA = sqlite3.pc

@@ -1,19 +0,0 @@
-#!/usr/bin/make
-#
-# If the configure script does not work, then this Makefile is available
-# as a backup.  Manually configure the variables below.
-#
-# Note:  This makefile works out-of-the-box on MacOS 10.2 (Jaguar)
-#
-CC = gcc
-CFLAGS = -O0 -I.
-LIBS = -lz
-COPTS += -D_BSD_SOURCE
-COPTS += -DSQLITE_ENABLE_LOCKING_STYLE=0
-COPTS += -DSQLITE_THREADSAFE=0
-COPTS += -DSQLITE_OMIT_LOAD_EXTENSION
-COPTS += -DSQLITE_WITHOUT_ZONEMALLOC
-COPTS += -DSQLITE_ENABLE_RTREE
-
-sqlite3:	shell.c sqlite3.c
-	$(CC) $(CFLAGS) $(COPTS) -o sqlite3 shell.c sqlite3.c $(LIBS)
@@ -21,14 +21,7 @@ TOP = .
 # Set this non-0 to enable full warnings (-W4, etc) when compiling.
 #
 !IFNDEF USE_FULLWARN
-USE_FULLWARN = 1
-!ENDIF
-
-# Set this non-0 to enable treating warnings as errors (-WX, etc) when
-# compiling.
-#
-!IFNDEF USE_FATAL_WARN
-USE_FATAL_WARN = 0
+USE_FULLWARN = 0
 !ENDIF

 # Set this non-0 to enable full runtime error checks (-RTC1, etc).  This
@@ -73,7 +66,7 @@ API_ARMOR = 0
 !IFNDEF NO_WARN
 !IF $(USE_FULLWARN)!=0
 NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4130 -wd4152 -wd4189 -wd4206
-NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706
+NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4305 -wd4306 -wd4702 -wd4706
 !ENDIF
 !ENDIF

@@ -210,12 +203,6 @@ OPTIMIZATIONS = 2
 SESSION = 0
 !ENDIF

-# Set this to non-0 to enable support for the rbu extension.
-#
-!IFNDEF RBU
-RBU = 0
-!ENDIF
-
 # Set the source code file to be used by executables and libraries when
 # they need the amalgamation.
 #
@@ -283,12 +270,6 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
@@ -301,13 +282,6 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1
 !ENDIF

-# Should the rbu extension be enabled?  If so, add compilation options
-# to enable it.
-#
-!IF $(RBU)!=0
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
-!ENDIF
-
 # These are the "extended" SQLite compilation options used when compiling for
 # the Windows 10 platform.
 #
@@ -445,9 +419,9 @@ UCRTLIBPATH = $(UCRTLIBPATH:\\=\)
 # will run on the platform that is doing the build.
 #
 !IF $(USE_FULLWARN)!=0
-BCC = $(NCC) -nologo -W4 -Fd$*.pdb $(CCOPTS) $(BCCOPTS)
+BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS)
 !ELSE
-BCC = $(NCC) -nologo -W3 -Fd$*.pdb $(CCOPTS) $(BCCOPTS)
+BCC = $(NCC) -nologo -W3 $(CCOPTS) $(BCCOPTS)
 !ENDIF

 # Check if assembly code listings should be generated for the source
@@ -480,12 +454,6 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
 TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
 !ENDIF

-# Check if warnings should be treated as errors when compiling.
-#
-!IF $(USE_FATAL_WARN)!=0
-TCC = $(TCC) -WX
-!ENDIF
-
 TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise
 RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)

@@ -580,7 +548,6 @@ SHELL_CORE_DEP =
 !ENDIF
 !ENDIF

-
 # This is the core library that the shell executable should link with.
 #
 !IFNDEF SHELL_CORE_LIB
@@ -665,10 +632,6 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
 !IF $(DEBUG)>2
 TCC = $(TCC) -DSQLITE_DEBUG=1
 RCC = $(RCC) -DSQLITE_DEBUG=1
-!IF $(DYNAMIC_SHELL)==0
-TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
-RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
-!ENDIF
 !ENDIF

 !IF $(DEBUG)>4 || $(OSTRACE)!=0
@@ -820,7 +783,7 @@ BCC = $(BCC) -Zi
 # Command line prefixes for compiling code, compiling resources,
 # linking, etc.
 #
-LTCOMPILE = $(TCC) -Fo$@ -Fd$*.pdb
+LTCOMPILE = $(TCC) -Fo$@
 LTRCOMPILE = $(RCC) -r
 LTLIB = lib.exe
 LTLINK = $(TCC) -Fe$@
@@ -828,7 +791,7 @@ LTLINK = $(TCC) -Fe$@
 # If requested, link to the RPCRT4 library.
 #
 !IF $(USE_RPCRT4_LIB)!=0
-LTLIBS = $(LTLIBS) rpcrt4.lib
+LTLINK = $(LTLINK) rpcrt4.lib
 !ENDIF

 # If a platform was set, force the linker to target that.
@@ -838,11 +801,6 @@ LTLIBS = $(LTLIBS) rpcrt4.lib
 !IFDEF PLATFORM
 LTLINKOPTS = /NOLOGO /MACHINE:$(PLATFORM)
 LTLIBOPTS = /NOLOGO /MACHINE:$(PLATFORM)
-!ELSEIF "$(VISUALSTUDIOVERSION)"=="12.0" || \
-        "$(VISUALSTUDIOVERSION)"=="14.0" || \
-        "$(VISUALSTUDIOVERSION)"=="15.0"
-LTLINKOPTS = /NOLOGO /MACHINE:x86
-LTLIBOPTS = /NOLOGO /MACHINE:x86
 !ELSE
 LTLINKOPTS = /NOLOGO
 LTLIBOPTS = /NOLOGO
@@ -952,25 +910,14 @@ LIBRESOBJS =
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 !ENDIF


 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-core:	dll shell
-
-# Targets that require the Tcl library.
-#
-tcl:	$(ALL_TCL_TARGETS)
-
-# This Makefile target builds all of the standard binaries.
-#
-all:	core tcl
+all:	dll shell

 # Dynamic link library section.
 #
@@ -990,11 +937,11 @@ Replace.exe:
 sqlite3.def:	Replace.exe $(LIBOBJ)
 	echo EXPORTS > sqlite3.def
 	dumpbin /all $(LIBOBJ) \
-		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup|rebaser|rbu)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
+		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
 		| sort >> sqlite3.def

-$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
+$(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)


@@ -1009,7 +956,7 @@ sqlite3.lo:	$(SQLITE3C)
 !IF $(USE_RC)!=0
 _HASHCHAR=^#
 !IF ![echo !IFNDEF VERSION > rcver.vc] && \
-    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| "%SystemRoot%\System32\find.exe" "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
+    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
    ![echo !ENDIF >> rcver.vc]
 !INCLUDE rcver.vc
 !ENDIF
@@ -12,7 +12,6 @@
 AC_PREREQ(2.61)
 AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org)
 AC_CONFIG_SRCDIR([sqlite3.c])
-AC_CONFIG_AUX_DIR([.])

 # Use automake.
 AM_INIT_AUTOMAKE([foreign])
@@ -29,7 +28,6 @@ AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
 AC_FUNC_STRERROR_R

 AC_CONFIG_FILES([Makefile sqlite3.pc])
-BUILD_CFLAGS=
 AC_SUBST(BUILD_CFLAGS)

 #-------------------------------------------------------------------------
@@ -57,9 +55,9 @@ AS_IF([ test x"$enable_editline" != xno ],[
    LIBS=""
    AC_SEARCH_LIBS([readline],[edit],[
      AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
-      READLINE_LIBS="$LIBS -ltinfo"
+      READLINE_LIBS=$LIBS
      enable_readline=no
-    ],[],[-ltinfo])
+    ])
    AS_UNSET(ac_cv_search_readline)
    LIBS=$sLIBS
  ])
@@ -87,11 +85,13 @@ AC_SUBST(READLINE_LIBS)
 AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING(
  [--enable-threadsafe], [build a thread-safe library [default=yes]])], 
  [], [enable_threadsafe=yes])
+THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
 if test x"$enable_threadsafe" != "xno"; then
-  BUILD_CFLAGS="$BUILD_CFLAGS -D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
+  THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
  AC_SEARCH_LIBS(pthread_create, pthread)
  AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
 fi
+AC_SUBST(THREADSAFE_FLAGS)
 #-----------------------------------------------------------------------

 #-----------------------------------------------------------------------
@@ -103,66 +103,36 @@ AC_ARG_ENABLE(dynamic-extensions, [AS_HELP_STRING(
 if test x"$enable_dynamic_extensions" != "xno"; then
  AC_SEARCH_LIBS(dlopen, dl)
 else
-  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_OMIT_LOAD_EXTENSION=1"
+  DYNAMIC_EXTENSION_FLAGS=-DSQLITE_OMIT_LOAD_EXTENSION=1
 fi
 AC_MSG_CHECKING([for whether to support dynamic extensions])
 AC_MSG_RESULT($enable_dynamic_extensions)
-#-----------------------------------------------------------------------
-
-#-----------------------------------------------------------------------
-#   --enable-fts4
-#
-AC_ARG_ENABLE(fts4, [AS_HELP_STRING(
-  [--enable-fts4], [include fts4 support [default=yes]])], 
-  [], [enable_fts4=yes])
-if test x"$enable_fts4" = "xyes"; then
-  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS4"
-fi
-#-----------------------------------------------------------------------
-
-#-----------------------------------------------------------------------
-#   --enable-fts3
-#
-AC_ARG_ENABLE(fts3, [AS_HELP_STRING(
-  [--enable-fts3], [include fts3 support [default=no]])], 
-  [], [])
-if test x"$enable_fts3" = "xyes" -a x"$enable_fts4" = "xno"; then
-  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS3"
-fi
+AC_SUBST(DYNAMIC_EXTENSION_FLAGS)
 #-----------------------------------------------------------------------

 #-----------------------------------------------------------------------
 #   --enable-fts5
 #
 AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
-  [--enable-fts5], [include fts5 support [default=yes]])], 
-  [], [enable_fts5=yes])
+  [--enable-fts5], [include fts5 support [default=no]])], 
+  [], [enable_fts5=no])
 if test x"$enable_fts5" = "xyes"; then
  AC_SEARCH_LIBS(log, m)
-  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS5"
+  FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
 fi
+AC_SUBST(FTS5_FLAGS)
 #-----------------------------------------------------------------------

 #-----------------------------------------------------------------------
 #   --enable-json1
 #
 AC_ARG_ENABLE(json1, [AS_HELP_STRING(
-  [--enable-json1], [include json1 support [default=yes]])], 
-  [],[enable_json1=yes])
+  [--enable-json1], [include json1 support [default=no]])], 
+  [], [enable_json1=no])
 if test x"$enable_json1" = "xyes"; then
-  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_JSON1"
-fi
-#-----------------------------------------------------------------------
-
-#-----------------------------------------------------------------------
-#   --enable-rtree
-#
-AC_ARG_ENABLE(rtree, [AS_HELP_STRING(
-  [--enable-rtree], [include rtree support [default=yes]])], 
-  [], [enable_rtree=yes])
-if test x"$enable_rtree" = "xyes"; then
-  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_GEOPOLY"
+  JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
 fi
+AC_SUBST(JSON1_FLAGS)
 #-----------------------------------------------------------------------

 #-----------------------------------------------------------------------
@@ -170,22 +140,11 @@ fi
 #
 AC_ARG_ENABLE(session, [AS_HELP_STRING(
  [--enable-session], [enable the session extension [default=no]])], 
-  [], [])
+  [], [enable_session=no])
 if test x"$enable_session" = "xyes"; then
-  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
-fi
-#-----------------------------------------------------------------------
-
-#-----------------------------------------------------------------------
-#   --enable-debug
-#
-AC_ARG_ENABLE(debug, [AS_HELP_STRING(
-  [--enable-debug], [build with debugging features enabled [default=no]])], 
-  [], [])
-if test x"$enable_debug" = "xyes"; then
-  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_DEBUG -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE"
-  CFLAGS="-g -O0"
+  SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
 fi
+AC_SUBST(SESSION_FLAGS)
 #-----------------------------------------------------------------------

 #-----------------------------------------------------------------------
@@ -204,12 +163,6 @@ AC_SUBST(EXTRA_SHELL_OBJ)
 #-----------------------------------------------------------------------

 AC_CHECK_FUNCS(posix_fallocate)
-AC_CHECK_HEADERS(zlib.h,[
-  AC_SEARCH_LIBS(deflate,z,[BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_HAVE_ZLIB"])
-])
-
-AC_SEARCH_LIBS(system,,,[SHELL_CFLAGS="-DSQLITE_NOHAVE_SYSTEM"])
-AC_SUBST(SHELL_CFLAGS)

 #-----------------------------------------------------------------------
 # UPDATE: Maybe it's better if users just set CFLAGS before invoking
@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so you can encode the package version directly into the source files.
 #-----------------------------------------------------------------------

-AC_INIT([sqlite], [3.32.0])
+AC_INIT([sqlite], [3.7.4])

 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.
@@ -153,7 +153,7 @@ Please `cd` to its location first.
 #
 #-------------------------------------------------------------------------

-PROJECT = tclsqlite3
+PROJECT = sqlite3
 !include "rules.vc"

 # nmakehelp -V <file> <tag> will search the file for tag, skips until a
@@ -162,15 +162,18 @@ PROJECT = tclsqlite3

 !if [echo REM = This file is generated from Makefile.vc > versions.vc]
 !endif
-# get project version from row "AC_INIT([sqlite], [3.x.y])"
+# get project version from row "AC_INIT([sqlite], [3.7.14])"
 !if [echo DOTVERSION = \>> versions.vc] \
-   && [nmakehlp -V ..\configure.ac AC_INIT >> versions.vc]
+   && [nmakehlp -V ..\configure.in AC_INIT >> versions.vc]
 !endif
 !include "versions.vc"

 VERSION         = $(DOTVERSION:.=)
 STUBPREFIX      = $(PROJECT)stub

+DLLOBJS = \
+	$(TMP_DIR)\tclsqlite3.obj
+
 #-------------------------------------------------------------------------
 # Target names and paths ( shouldn't need changing )
 #-------------------------------------------------------------------------
@@ -179,7 +182,7 @@ BINROOT		= .
 ROOT            = ..

 PRJIMPLIB	= $(OUT_DIR)\$(PROJECT)$(VERSION)$(SUFX).lib
-PRJLIBNAME	= $(PROJECT).$(EXT)
+PRJLIBNAME	= $(PROJECT)$(VERSION)$(SUFX).$(EXT)
 PRJLIB		= $(OUT_DIR)\$(PRJLIBNAME)

 PRJSTUBLIBNAME	= $(STUBPREFIX)$(VERSION).lib
@@ -201,17 +204,6 @@ DOCDIR		= $(ROOT)\doc
 TOOLSDIR	= $(ROOT)\tools
 COMPATDIR	= $(ROOT)\compat

-### Figure out where the primary source code file(s) is/are.
-!if exist("$(ROOT)\..\..\sqlite3.c") && exist("$(ROOT)\..\..\src\tclsqlite.c")
-SQL_INCLUDES = -I"$(ROOT)\..\.."
-SQLITE_SRCDIR = $(ROOT)\..\..
-TCLSQLITE_SRCDIR = $(ROOT)\..\..\src
-DLLOBJS = $(TMP_DIR)\sqlite3.obj $(TMP_DIR)\tclsqlite.obj
-!else
-TCLSQLITE_SRCDIR = $(ROOT)\generic
-DLLOBJS = $(TMP_DIR)\tclsqlite3.obj
-!endif
-
 #---------------------------------------------------------------------
 # Compile flags
 #---------------------------------------------------------------------
@@ -231,7 +223,7 @@ cdebug	= -Z7 -WX -Od -GZ
 !endif

 ### Declarations common to all compiler options
-cflags = -nologo -c -W3 -D_CRT_SECURE_NO_WARNINGS -YX -Fp$(TMP_DIR)^\
+cflags = -nologo -c -W3 -YX -Fp$(TMP_DIR)^\

 !if $(MSVCRT)
 !if $(DEBUG)
@@ -247,8 +239,8 @@ crt = -MT
 !endif
 !endif

-INCLUDES	= $(SQL_INCLUDES) $(TCL_INCLUDES) -I"$(WINDIR)" \
-                  -I"$(GENERICDIR)" -I"$(ROOT)\.."
+INCLUDES	= $(TCL_INCLUDES) -I"$(WINDIR)" -I"$(GENERICDIR)" \
+                  -I"$(ROOT)\.."
 BASE_CLFAGS	= $(cflags) $(cdebug) $(crt) $(INCLUDES) \
                  -DSQLITE_3_SUFFIX_ONLY=1 -DSQLITE_ENABLE_RTREE=1 \
                  -DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1
@@ -349,17 +341,20 @@ $(PRJSTUBLIB): $(PRJSTUBOBJS)
 # Implicit rules
 #---------------------------------------------------------------------

-$(TMP_DIR)\sqlite3.obj:		$(SQLITE_SRCDIR)\sqlite3.c
-	$(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ \
-		-c $(SQLITE_SRCDIR)\sqlite3.c
+{$(WINDIR)}.c{$(TMP_DIR)}.obj::
+    $(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ @<<
+$<
+<<

-$(TMP_DIR)\tclsqlite.obj:	$(TCLSQLITE_SRCDIR)\tclsqlite.c
-	$(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ \
-		-c $(TCLSQLITE_SRCDIR)\tclsqlite.c
+{$(GENERICDIR)}.c{$(TMP_DIR)}.obj::
+    $(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ @<<
+$<
+<<

-$(TMP_DIR)\tclsqlite3.obj:	$(TCLSQLITE_SRCDIR)\tclsqlite3.c
-	$(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ \
-		-c $(TCLSQLITE_SRCDIR)\tclsqlite3.c
+{$(COMPATDIR)}.c{$(TMP_DIR)}.obj::
+    $(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ @<<
+$<
+<<

 {$(WINDIR)}.rc{$(TMP_DIR)}.res:
 	$(rc32) -fo $@ -r -i "$(GENERICDIR)" -D__WIN32__ \
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.32.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.16.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.32.0'
-PACKAGE_STRING='sqlite 3.32.0'
+PACKAGE_VERSION='3.16.0'
+PACKAGE_STRING='sqlite 3.16.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -772,7 +772,6 @@ LIBOBJS
 BUILD_CFLAGS
 USE_GCOV
 OPT_FEATURE_FLAGS
-HAVE_ZLIB
 USE_AMALGAMATION
 TARGET_DEBUG
 TARGET_HAVE_EDITLINE
@@ -906,13 +905,10 @@ enable_amalgamation
 enable_load_extension
 enable_memsys5
 enable_memsys3
-enable_all
 enable_fts3
 enable_fts4
 enable_fts5
 enable_json1
-enable_update_limit
-enable_geopoly
 enable_rtree
 enable_session
 enable_gcov
@@ -1467,7 +1463,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.32.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.16.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1532,7 +1528,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.32.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.16.0:";;
   esac
  cat <<\_ACEOF

@@ -1560,13 +1556,10 @@ Optional Features:
                          Disable loading of external extensions
  --enable-memsys5        Enable MEMSYS5
  --enable-memsys3        Enable MEMSYS3
-  --enable-all            Enable FTS4, FTS5, Geopoly, JSON, RTree, Sessions
  --enable-fts3           Enable the FTS3 extension
  --enable-fts4           Enable the FTS4 extension
  --enable-fts5           Enable the FTS5 extension
  --enable-json1          Enable the JSON1 extension
-  --enable-update-limit   Enable the UPDATE/DELETE LIMIT clause
-  --enable-geopoly        Enable the GEOPOLY extension
  --enable-rtree          Enable the RTREE extension
  --enable-session        Enable the SESSION extension
  --enable-gcov           Enable coverage testing using gcov
@@ -1659,7 +1652,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.32.0
+sqlite configure 3.16.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2078,7 +2071,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.32.0, which was
+It was created by sqlite $as_me 3.16.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -3936,13 +3929,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
  lt_cv_nm_interface="BSD nm"
  echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3939: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3932: $ac_compile\"" >&5)
  (eval "$ac_compile" 2>conftest.err)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3942: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3945: output\"" >&5)
+  (eval echo "\"\$as_me:3938: output\"" >&5)
  cat conftest.out >&5
  if $GREP 'External.*some_variable' conftest.out > /dev/null; then
    lt_cv_nm_interface="MS dumpbin"
@@ -5148,7 +5141,7 @@ ia64-*-hpux*)
  ;;
 *-*-irix6*)
  # Find out which ABI we are using.
-  echo '#line 5151 "configure"' > conftest.$ac_ext
+  echo '#line 5144 "configure"' > conftest.$ac_ext
  if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  (eval $ac_compile) 2>&5
  ac_status=$?
@@ -6673,11 +6666,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6676: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6669: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:6680: \$? = $ac_status" >&5
+   echo "$as_me:6673: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7012,11 +7005,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7015: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7008: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:7019: \$? = $ac_status" >&5
+   echo "$as_me:7012: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7117,11 +7110,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7120: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7113: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7124: \$? = $ac_status" >&5
+   echo "$as_me:7117: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -7172,11 +7165,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7175: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7168: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7179: \$? = $ac_status" >&5
+   echo "$as_me:7172: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -9552,7 +9545,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9555 "configure"
+#line 9548 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -9648,7 +9641,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9651 "configure"
+#line 9644 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -10309,7 +10302,7 @@ USE_AMALGAMATION=1
 # if not, then we fall back to plain tclsh.
 # TODO: try other versions before falling back?
 #
-for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh
+for ac_prog in tclsh8.6 tclsh8.5 tclsh
 do
  # Extract the first word of "$ac_prog", so it can be a program name with args.
 set dummy $ac_prog; ac_word=$2
@@ -10459,6 +10452,8 @@ fi
 # Check whether --enable-threadsafe was given.
 if test "${enable_threadsafe+set}" = set; then :
  enableval=$enable_threadsafe;
+else
+  enable_threadsafe=yes
 fi

 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support threadsafe operation" >&5
@@ -11251,11 +11246,13 @@ fi
 # check for debug enabled
 # Check whether --enable-debug was given.
 if test "${enable_debug+set}" = set; then :
-  enableval=$enable_debug;
+  enableval=$enable_debug; use_debug=$enableval
+else
+  use_debug=no
 fi

-if test "${enable_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
+if test "${use_debug}" = "yes" ; then
+  TARGET_DEBUG="-DSQLITE_DEBUG=1"
 else
  TARGET_DEBUG="-DNDEBUG"
 fi
@@ -11265,98 +11262,26 @@ fi
 # See whether we should use the amalgamation to build
 # Check whether --enable-amalgamation was given.
 if test "${enable_amalgamation+set}" = set; then :
-  enableval=$enable_amalgamation;
+  enableval=$enable_amalgamation; use_amalgamation=$enableval
+else
+  use_amalgamation=yes
 fi

-if test "${enable_amalgamation}" == "no" ; then
+if test "${use_amalgamation}" != "yes" ; then
  USE_AMALGAMATION=0
 fi


-#########
-# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
-for ac_header in zlib.h
-do :
-  ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default"
-if test "x$ac_cv_header_zlib_h" = xyes; then :
-  cat >>confdefs.h <<_ACEOF
-#define HAVE_ZLIB_H 1
-_ACEOF
-
-fi
-
-done
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing deflate" >&5
-$as_echo_n "checking for library containing deflate... " >&6; }
-if ${ac_cv_search_deflate+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-  ac_func_search_save_LIBS=$LIBS
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-
-/* Override any GCC internal prototype to avoid an error.
-   Use char because int might match the return type of a GCC
-   builtin and then its argument prototype would still apply.  */
-#ifdef __cplusplus
-extern "C"
-#endif
-char deflate ();
-int
-main ()
-{
-return deflate ();
-  ;
-  return 0;
-}
-_ACEOF
-for ac_lib in '' z; do
-  if test -z "$ac_lib"; then
-    ac_res="none required"
-  else
-    ac_res=-l$ac_lib
-    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
-  fi
-  if ac_fn_c_try_link "$LINENO"; then :
-  ac_cv_search_deflate=$ac_res
-fi
-rm -f core conftest.err conftest.$ac_objext \
-    conftest$ac_exeext
-  if ${ac_cv_search_deflate+:} false; then :
-  break
-fi
-done
-if ${ac_cv_search_deflate+:} false; then :
-
-else
-  ac_cv_search_deflate=no
-fi
-rm conftest.$ac_ext
-LIBS=$ac_func_search_save_LIBS
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_deflate" >&5
-$as_echo "$ac_cv_search_deflate" >&6; }
-ac_res=$ac_cv_search_deflate
-if test "$ac_res" != no; then :
-  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
-  HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"
-else
-  HAVE_ZLIB=""
-fi
-
-
-
 #########
 # See whether we should allow loadable extensions
 # Check whether --enable-load-extension was given.
 if test "${enable_load_extension+set}" = set; then :
-  enableval=$enable_load_extension;
+  enableval=$enable_load_extension; use_loadextension=$enableval
 else
-  enable_load_extension=yes
+  use_loadextension=yes
 fi

-if test "${enable_load_extension}" = "yes" ; then
+if test "${use_loadextension}" = "yes" ; then
  OPT_FEATURE_FLAGS=""
  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing dlopen" >&5
 $as_echo_n "checking for library containing dlopen... " >&6; }
@@ -11423,13 +11348,15 @@ fi
 #
 # Check whether --enable-memsys5 was given.
 if test "${enable_memsys5+set}" = set; then :
-  enableval=$enable_memsys5;
+  enableval=$enable_memsys5; enable_memsys5=yes
+else
+  enable_memsys5=no
 fi

 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5
 $as_echo_n "checking whether to support MEMSYS5... " >&6; }
 if test "${enable_memsys5}" = "yes"; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5"
  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 $as_echo "yes" >&6; }
 else
@@ -11438,13 +11365,15 @@ $as_echo "no" >&6; }
 fi
 # Check whether --enable-memsys3 was given.
 if test "${enable_memsys3+set}" = set; then :
-  enableval=$enable_memsys3;
+  enableval=$enable_memsys3; enable_memsys3=yes
+else
+  enable_memsys3=no
 fi

 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5
 $as_echo_n "checking whether to support MEMSYS3... " >&6; }
 if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3"
  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 $as_echo "yes" >&6; }
 else
@@ -11452,32 +11381,27 @@ else
 $as_echo "no" >&6; }
 fi

-########
-# The --enable-extensions argument is short-hand to enable
-# multiple extensions.
-# Check whether --enable-all was given.
-if test "${enable_all+set}" = set; then :
-  enableval=$enable_all;
-fi
-
-
 #########
 # See whether we should enable Full Text Search extensions
 # Check whether --enable-fts3 was given.
 if test "${enable_fts3+set}" = set; then :
-  enableval=$enable_fts3;
+  enableval=$enable_fts3; enable_fts3=yes
+else
+  enable_fts3=no
 fi

 if test "${enable_fts3}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3"
 fi
 # Check whether --enable-fts4 was given.
 if test "${enable_fts4+set}" = set; then :
-  enableval=$enable_fts4;
+  enableval=$enable_fts4; enable_fts4=yes
+else
+  enable_fts4=no
 fi

-if test "${enable_fts4}" = "yes" -o "${enable_all}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
+if test "${enable_fts4}" = "yes" ; then
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4"
  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
 $as_echo_n "checking for library containing log... " >&6; }
 if ${ac_cv_search_log+:} false; then :
@@ -11537,11 +11461,13 @@ fi
 fi
 # Check whether --enable-fts5 was given.
 if test "${enable_fts5+set}" = set; then :
-  enableval=$enable_fts5;
+  enableval=$enable_fts5; enable_fts5=yes
+else
+  enable_fts5=no
 fi

-if test "${enable_fts5}" = "yes" -o "${enable_all}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
+if test "${enable_fts5}" = "yes" ; then
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5"
  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
 $as_echo_n "checking for library containing log... " >&6; }
 if ${ac_cv_search_log+:} false; then :
@@ -11604,64 +11530,44 @@ fi
 # See whether we should enable JSON1
 # Check whether --enable-json1 was given.
 if test "${enable_json1+set}" = set; then :
-  enableval=$enable_json1;
-fi
-
-if test "${enable_json1}" = "yes" -o "${enable_all}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
-fi
-
-#########
-# See whether we should enable the LIMIT clause on UPDATE and DELETE
-# statements.
-# Check whether --enable-update-limit was given.
-if test "${enable_update_limit+set}" = set; then :
-  enableval=$enable_update_limit;
-fi
-
-if test "${enable_udlimit}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
-fi
-
-#########
-# See whether we should enable GEOPOLY
-# Check whether --enable-geopoly was given.
-if test "${enable_geopoly+set}" = set; then :
-  enableval=$enable_geopoly; enable_geopoly=yes
+  enableval=$enable_json1; enable_json1=yes
 else
-  enable_geopoly=no
+  enable_json1=no
 fi

-if test "${enable_geopoly}" = "yes" -o "${enable_all}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY"
-  enable_rtree=yes
+if test "${enable_json1}" = "yes" ; then
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1"
 fi

 #########
 # See whether we should enable RTREE
 # Check whether --enable-rtree was given.
 if test "${enable_rtree+set}" = set; then :
-  enableval=$enable_rtree;
+  enableval=$enable_rtree; enable_rtree=yes
+else
+  enable_rtree=no
 fi

 if test "${enable_rtree}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
 fi

 #########
 # See whether we should enable the SESSION extension
 # Check whether --enable-session was given.
 if test "${enable_session+set}" = set; then :
-  enableval=$enable_session;
+  enableval=$enable_session; enable_session=yes
+else
+  enable_session=no
 fi

-if test "${enable_session}" = "yes" -o "${enable_all}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
+if test "${enable_session}" = "yes" ; then
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION"
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK"
 fi

 #########
-# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter
+# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
 for option in $CFLAGS $CPPFLAGS
 do
  case $option in
@@ -11715,7 +11621,9 @@ BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
 # See whether we should use GCOV
 # Check whether --enable-gcov was given.
 if test "${enable_gcov+set}" = set; then :
-  enableval=$enable_gcov;
+  enableval=$enable_gcov; use_gcov=$enableval
+else
+  use_gcov=no
 fi

 if test "${use_gcov}" = "yes" ; then
@@ -12243,7 +12151,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.32.0, which was
+This file was extended by sqlite $as_me 3.16.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12309,7 +12217,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.32.0
+sqlite config.status 3.16.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -120,7 +120,7 @@ USE_AMALGAMATION=1
 # if not, then we fall back to plain tclsh.
 # TODO: try other versions before falling back?
 # 
-AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none)
+AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh8.5 tclsh], none)
 if test "$TCLSH_CMD" = "none"; then
  # If we can't find a local tclsh, then building the amalgamation will fail.
  # We act as though --disable-amalgamation has been used.
@@ -182,7 +182,7 @@ AC_SUBST(BUILD_CC)
 # Do we want to support multithreaded use of sqlite
 #
 AC_ARG_ENABLE(threadsafe, 
-AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]))
+AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]),,enable_threadsafe=yes)
 AC_MSG_CHECKING([whether to support threadsafe operation])
 if test "$enable_threadsafe" = "no"; then
  SQLITE_THREADSAFE=0
@@ -557,9 +557,10 @@ AC_SEARCH_LIBS(fdatasync, [rt])

 #########
 # check for debug enabled
-AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
-if test "${enable_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
+AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
+      [use_debug=$enableval],[use_debug=no])
+if test "${use_debug}" = "yes" ; then
+  TARGET_DEBUG="-DSQLITE_DEBUG=1"
 else
  TARGET_DEBUG="-DNDEBUG"
 fi
@@ -568,23 +569,19 @@ AC_SUBST(TARGET_DEBUG)
 #########
 # See whether we should use the amalgamation to build
 AC_ARG_ENABLE(amalgamation, AC_HELP_STRING([--disable-amalgamation],
-      [Disable the amalgamation and instead build all files separately]))
-if test "${enable_amalgamation}" == "no" ; then
+      [Disable the amalgamation and instead build all files separately]),
+      [use_amalgamation=$enableval],[use_amalgamation=yes])
+if test "${use_amalgamation}" != "yes" ; then
  USE_AMALGAMATION=0
 fi
 AC_SUBST(USE_AMALGAMATION)

-#########
-# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
-AC_CHECK_HEADERS(zlib.h)
-AC_SEARCH_LIBS(deflate, z, [HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"], [HAVE_ZLIB=""])
-AC_SUBST(HAVE_ZLIB)
-
 #########
 # See whether we should allow loadable extensions
 AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension],
-      [Disable loading of external extensions]),,[enable_load_extension=yes])
-if test "${enable_load_extension}" = "yes" ; then
+      [Disable loading of external extensions]),
+      [use_loadextension=$enableval],[use_loadextension=yes])
+if test "${use_loadextension}" = "yes" ; then
  OPT_FEATURE_FLAGS=""
  AC_SEARCH_LIBS(dlopen, dl)
 else
@@ -595,95 +592,79 @@ fi
 # Do we want to support memsys3 and/or memsys5
 #
 AC_ARG_ENABLE(memsys5, 
-  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]))
+  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]),
+  [enable_memsys5=yes],[enable_memsys5=no])
 AC_MSG_CHECKING([whether to support MEMSYS5])
 if test "${enable_memsys5}" = "yes"; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5"
  AC_MSG_RESULT([yes])
 else
  AC_MSG_RESULT([no])
 fi
 AC_ARG_ENABLE(memsys3, 
-  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]))
+  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]),
+  [enable_memsys3=yes],[enable_memsys3=no])
 AC_MSG_CHECKING([whether to support MEMSYS3])
 if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3"
  AC_MSG_RESULT([yes])
 else
  AC_MSG_RESULT([no])
 fi

-########
-# The --enable-extensions argument is short-hand to enable
-# multiple extensions.
-AC_ARG_ENABLE(all, AC_HELP_STRING([--enable-all],
-      [Enable FTS4, FTS5, Geopoly, JSON, RTree, Sessions]))
-
 #########
 # See whether we should enable Full Text Search extensions
 AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
-      [Enable the FTS3 extension]))
+      [Enable the FTS3 extension]),
+      [enable_fts3=yes],[enable_fts3=no])
 if test "${enable_fts3}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3"
 fi
 AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
-      [Enable the FTS4 extension]))
-if test "${enable_fts4}" = "yes" -o "${enable_all}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
+      [Enable the FTS4 extension]),
+      [enable_fts4=yes],[enable_fts4=no])
+if test "${enable_fts4}" = "yes" ; then
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4"
  AC_SEARCH_LIBS([log],[m])
 fi
 AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
-      [Enable the FTS5 extension]))
-if test "${enable_fts5}" = "yes" -o "${enable_all}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
+      [Enable the FTS5 extension]),
+      [enable_fts5=yes],[enable_fts5=no])
+if test "${enable_fts5}" = "yes" ; then
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5"
  AC_SEARCH_LIBS([log],[m])
 fi

 #########
 # See whether we should enable JSON1
-AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],[Enable the JSON1 extension]))
-if test "${enable_json1}" = "yes" -o "${enable_all}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
-fi
-
-#########
-# See whether we should enable the LIMIT clause on UPDATE and DELETE
-# statements.
-AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
-      [Enable the UPDATE/DELETE LIMIT clause]))
-if test "${enable_udlimit}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
-fi
-
-#########
-# See whether we should enable GEOPOLY
-AC_ARG_ENABLE(geopoly, AC_HELP_STRING([--enable-geopoly],
-      [Enable the GEOPOLY extension]),
-      [enable_geopoly=yes],[enable_geopoly=no])
-if test "${enable_geopoly}" = "yes" -o "${enable_all}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY"
-  enable_rtree=yes
+AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],
+      [Enable the JSON1 extension]),
+      [enable_json1=yes],[enable_json1=no])
+if test "${enable_json1}" = "yes" ; then
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1"
 fi

 #########
 # See whether we should enable RTREE
 AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
-      [Enable the RTREE extension]))
+      [Enable the RTREE extension]),
+      [enable_rtree=yes],[enable_rtree=no])
 if test "${enable_rtree}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
 fi

 #########
 # See whether we should enable the SESSION extension
 AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
-      [Enable the SESSION extension]))
-if test "${enable_session}" = "yes" -o "${enable_all}" = "yes" ; then
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
-  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
+      [Enable the SESSION extension]),
+      [enable_session=yes],[enable_session=no])
+if test "${enable_session}" = "yes" ; then
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION"
+  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK"
 fi

 #########
-# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter
+# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
 for option in $CFLAGS $CPPFLAGS
 do
  case $option in
@@ -736,7 +717,8 @@ BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
 #########
 # See whether we should use GCOV
 AC_ARG_ENABLE(gcov, AC_HELP_STRING([--enable-gcov],
-      [Enable coverage testing using gcov]))
+      [Enable coverage testing using gcov]),
+      [use_gcov=$enableval],[use_gcov=no])
 if test "${use_gcov}" = "yes" ; then
  USE_GCOV=1
 else
@@ -1,87 +0,0 @@
-
-SQLite's OS layer contains the following definitions used in F2FS related
-calls:
-
-#define F2FS_IOCTL_MAGIC        0xf5
-#define F2FS_IOC_START_ATOMIC_WRITE     _IO(F2FS_IOCTL_MAGIC, 1)
-#define F2FS_IOC_COMMIT_ATOMIC_WRITE    _IO(F2FS_IOCTL_MAGIC, 2)
-#define F2FS_IOC_START_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 3)
-#define F2FS_IOC_ABORT_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 5)
-#define F2FS_IOC_GET_FEATURES           _IOR(F2FS_IOCTL_MAGIC, 12, u32)
-#define F2FS_FEATURE_ATOMIC_WRITE       0x0004
-
-After opening a database file on Linux (including Android), SQLite determines
-whether or not a file supports F2FS atomic commits as follows:
-
-  u32 flags = 0;
-  rc = ioctl(fd, F2FS_IOC_GET_FEATURES, &flags);
-  if( rc==0 && (flags & F2FS_FEATURE_ATOMIC_WRITE) ){
-    /* File supports F2FS atomic commits */
-  }else{
-    /* File does NOT support F2FS atomic commits */
-  }
-
-where "fd" is the file-descriptor open on the database file.
-
-Usually, when writing to a database file that supports atomic commits, SQLite
-accumulates the entire transaction in heap memory, deferring all writes to the
-db file until the transaction is committed.
-
-When it is time to commit a transaction on a file that supports atomic
-commits, SQLite does:
-
-  /* Take an F_WRLCK lock on the database file. This prevents any other
-  ** SQLite clients from reading or writing the file until the lock
-  ** is released.  */
-  rc = fcntl(fd, F_SETLK, ...);
-  if( rc!=0 ) goto failed;
-
-  rc = ioctl(fd, F2FS_IOC_START_ATOMIC_WRITE);
-  if( rc!=0 ) goto fallback_to_legacy_journal_commit;
-
-  foreach (dirty page){
-    rc = write(fd, ...dirty page...);
-    if( rc!=0 ){
-      ioctl(fd, F2FS_IOC_ABORT_VOLATILE_WRITE);
-      goto fallback_to_legacy_journal_commit;
-    }
-  }
-
-  rc = ioctl(fd, F2FS_IOC_COMMIT_ATOMIC_WRITE);
-  if( rc!=0 ){
-    ioctl(fd, F2FS_IOC_ABORT_VOLATILE_WRITE);
-    goto fallback_to_legacy_journal_commit;
-  }
-
-  /* If we get there, the transaction has been successfully 
-  ** committed to persistent storage. The following call
-  ** relinquishes the F_WRLCK lock.  */
-  fcntl(fd, F_SETLK, ...);
-
-Assumptions:
-
-1. After either of the F2FS_IOC_ABORT_VOLATILE_WRITE calls return,
-   the database file is in the state that it was in before
-   F2FS_IOC_START_ATOMIC_WRITE was invoked. Even if the ioctl()
-   fails - we're ignoring the return code.
-
-   This is true regardless of the type of error that occurred in
-   ioctl() or write().
-
-2. If the system fails before the F2FS_IOC_COMMIT_ATOMIC_WRITE is
-   completed, then following a reboot the database file is in the
-   state that it was in before F2FS_IOC_START_ATOMIC_WRITE was invoked.
-   Or, if the write was commited right before the system failed, in a 
-   state indicating that all write() calls were successfully committed
-   to persistent storage before the failure occurred.
-
-3. If the process crashes before the F2FS_IOC_COMMIT_ATOMIC_WRITE is
-   completed then the file is automatically restored to the state that
-   it was in before F2FS_IOC_START_ATOMIC_WRITE was called. This occurs
-   before the posix advisory lock is automatically dropped - there is
-   no chance that another client will be able to read the file in a
-   half-committed state before the rollback operation occurs.
-
-
-
-
@@ -2,12 +2,12 @@
 <head>
 <title>The Lemon Parser Generator</title>
 </head>
-<body bgcolor='white'>
-<h1 align='center'>The Lemon Parser Generator</h1>
+<body bgcolor=white>
+<h1 align=center>The Lemon Parser Generator</h1>

 <p>Lemon is an LALR(1) parser generator for C.
 It does the same job as "bison" and "yacc".
-But Lemon is not a bison or yacc clone.  Lemon
+But lemon is not a bison or yacc clone.  Lemon
 uses a different grammar syntax which is designed to
 reduce the number of coding errors.  Lemon also uses a
 parsing engine that is faster than yacc and
@@ -16,33 +16,13 @@ bison and which is both reentrant and threadsafe.
 has also been updated so that it too can generate a
 reentrant and threadsafe parser.)
 Lemon also implements features that can be used
-to eliminate resource leaks, making it suitable for use
+to eliminate resource leaks, making is suitable for use
 in long-running programs such as graphical user interfaces
 or embedded controllers.</p>

 <p>This document is an introduction to the Lemon
 parser generator.</p>

-<h2>Security Note</h2>
-
-<p>The language parser code created by Lemon is very robust and
-is well-suited for use in internet-facing applications that need to
-safely process maliciously crafted inputs.
-
-<p>The "lemon.exe" command-line tool itself works great when given a valid
-input grammar file and almost always gives helpful
-error messages for malformed inputs.  However,  it is possible for
-a malicious user to craft a grammar file that will cause 
-lemon.exe to crash.
-We do not see this as a problem, as lemon.exe is not intended to be used
-with hostile inputs.
-To summarize:</p>
-
-<ul>
-<li>Parser code generated by lemon &rarr; Robust and secure
-<li>The "lemon.exe" command line tool itself &rarr; Not so much
-</ul>
-
 <h2>Theory of Operation</h2>

 <p>The main goal of Lemon is to translate a context free grammar (CFG)
@@ -58,8 +38,8 @@ Lemon comes with a default parser template which works fine for most
 applications.  But the user is free to substitute a different parser
 template if desired.</p>

-<p>Depending on command-line options, Lemon will generate up to
-three output files.
+<p>Depending on command-line options, Lemon will generate between
+one and three files of outputs.
 <ul>
 <li>C code to implement the parser.
 <li>A header file defining an integer ID for each terminal symbol.
@@ -90,23 +70,17 @@ the states used by the parser automaton.</p>
 You can obtain a list of the available command-line options together
 with a brief explanation of what each does by typing
 <pre>
-   lemon "-?"
+   lemon -?
 </pre>
 As of this writing, the following command-line options are supported:
 <ul>
 <li><b>-b</b>
 Show only the basis for each parser state in the report file.
 <li><b>-c</b>
-Do not compress the generated action tables.  The parser will be a
-little larger and slower, but it will detect syntax errors sooner.
-<li><b>-d</b><i>directory</i>
-Write all output files into <i>directory</i>.  Normally, output files
-are written into the directory that contains the input grammar file.
+Do not compress the generated action tables.
 <li><b>-D<i>name</i></b>
-Define C preprocessor macro <i>name</i>.  This macro is usable by
-"<tt><a href='#pifdef'>%ifdef</a></tt>" and
-"<tt><a href='#pifdef'>%ifndef</a></tt>" lines
-in the grammar file.
+Define C preprocessor macro <i>name</i>.  This macro is useable by
+"%ifdef" lines in the grammar file.
 <li><b>-g</b>
 Do not generate a parser.  Instead write the input grammar to standard
 output with all comments, actions, and other extraneous text removed.
@@ -114,16 +88,16 @@ output with all comments, actions, and other extraneous text removed.
 Omit "#line" directives in the generated parser C code.
 <li><b>-m</b>
 Cause the output C source code to be compatible with the "makeheaders"
-program.
+program. 
 <li><b>-p</b>
-Display all conflicts that are resolved by
+Display all conflicts that are resolved by 
 <a href='#precrules'>precedence rules</a>.
 <li><b>-q</b>
 Suppress generation of the report file.
 <li><b>-r</b>
 Do not sort or renumber the parser states as part of optimization.
 <li><b>-s</b>
-Show parser statistics before exiting.
+Show parser statistics before existing.
 <li><b>-T<i>file</i></b>
 Use <i>file</i> as the template for the generated C-code parser implementation.
 <li><b>-x</b>
@@ -171,7 +145,7 @@ once for each token:
 </pre>
 The first argument to the Parse() routine is the pointer returned by
 ParseAlloc().
-The second argument is a small positive integer that tells the parser the
+The second argument is a small positive integer that tells the parse the
 type of the next token in the data stream.
 There is one token type for each terminal symbol in the grammar.
 The gram.h file generated by Lemon contains #define statements that
@@ -179,7 +153,7 @@ map symbolic terminal symbol names into appropriate integer values.
 A value of 0 for the second argument is a special flag to the
 parser to indicate that the end of input has been reached.
 The third argument is the value of the given token.  By default,
-the type of the third argument is "void*", but the grammar will
+the type of the third argument is integer, but the grammar will
 usually redefine this type to be some kind of structure.
 Typically the second argument will be a broad category of tokens
 such as "identifier" or "number" and the third argument will
@@ -187,7 +161,7 @@ be the name of the identifier or the value of the number.</p>

 <p>The Parse() function may have either three or four arguments,
 depending on the grammar.  If the grammar specification file requests
-it (via the <tt><a href='#extraarg'>%extra_argument</a></tt> directive),
+it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>),
 the Parse() function will have a fourth parameter that can be
 of any type chosen by the programmer.  The parser doesn't do anything
 with this argument except to pass it through to action routines.
@@ -197,20 +171,20 @@ to the action routines without having to use global variables.</p>
 <p>A typical use of a Lemon parser might look something like the
 following:
 <pre>
-    1 ParseTree *ParseFile(const char *zFilename){
-    2    Tokenizer *pTokenizer;
-    3    void *pParser;
-    4    Token sToken;
-    5    int hTokenId;
-    6    ParserState sState;
-    7
-    8    pTokenizer = TokenizerCreate(zFilename);
-    9    pParser = ParseAlloc( malloc );
-   10    InitParserState(&amp;sState);
-   11    while( GetNextToken(pTokenizer, &amp;hTokenId, &amp;sToken) ){
-   12       Parse(pParser, hTokenId, sToken, &amp;sState);
+   01 ParseTree *ParseFile(const char *zFilename){
+   02    Tokenizer *pTokenizer;
+   03    void *pParser;
+   04    Token sToken;
+   05    int hTokenId;
+   06    ParserState sState;
+   07
+   08    pTokenizer = TokenizerCreate(zFilename);
+   09    pParser = ParseAlloc( malloc );
+   10    InitParserState(&sState);
+   11    while( GetNextToken(pTokenizer, &hTokenId, &sToken) ){
+   12       Parse(pParser, hTokenId, sToken, &sState);
   13    }
-   14    Parse(pParser, 0, sToken, &amp;sState);
+   14    Parse(pParser, 0, sToken, &sState);
   15    ParseFree(pParser, free );
   16    TokenizerFree(pTokenizer);
   17    return sState.treeRoot;
@@ -223,10 +197,10 @@ simple.)
 We assume the existence of some kind of tokenizer which is created
 using TokenizerCreate() on line 8 and deleted by TokenizerFree()
 on line 16.  The GetNextToken() function on line 11 retrieves the
-next token from the input file and puts its type in the
+next token from the input file and puts its type in the 
 integer variable hTokenId.  The sToken variable is assumed to be
 some kind of structure that contains details about each token,
-such as its complete text, what line it occurs on, etc.</p>
+such as its complete text, what line it occurs on, etc. </p>

 <p>This example also assumes the existence of structure of type
 ParserState that holds state information about a particular parse.
@@ -243,7 +217,7 @@ tree.</p>
 <pre>
   ParseFile(){
      pParser = ParseAlloc( malloc );
-      while( GetNextToken(pTokenizer,&amp;hTokenId, &amp;sToken) ){
+      while( GetNextToken(pTokenizer,&hTokenId, &sToken) ){
         Parse(pParser, hTokenId, sToken);
      }
      Parse(pParser, 0, sToken);
@@ -303,25 +277,25 @@ specifies additional information Lemon requires to do its job.
 Most of the work in using Lemon is in writing an appropriate
 grammar file.</p>

-<p>The grammar file for Lemon is, for the most part, free format.
+<p>The grammar file for lemon is, for the most part, free format.
 It does not have sections or divisions like yacc or bison.  Any
 declaration can occur at any point in the file.
 Lemon ignores whitespace (except where it is needed to separate
-tokens), and it honors the same commenting conventions as C and C++.</p>
+tokens) and it honors the same commenting conventions as C and C++.</p>

 <h3>Terminals and Nonterminals</h3>

 <p>A terminal symbol (token) is any string of alphanumeric
 and/or underscore characters
-that begins with an uppercase letter.
+that begins with an upper case letter.
 A terminal can contain lowercase letters after the first character,
-but the usual convention is to make terminals all uppercase.
+but the usual convention is to make terminals all upper case.
 A nonterminal, on the other hand, is any string of alphanumeric
-and underscore characters than begins with a lowercase letter.
-Again, the usual convention is to make nonterminals use all lowercase
-letters.</p>
+and underscore characters than begins with a lower case letter.
+Again, the usual convention is to make nonterminals use all lower
+case letters.</p>

-<p>In Lemon, terminal and nonterminal symbols do not need to
+<p>In Lemon, terminal and nonterminal symbols do not need to 
 be declared or identified in a separate section of the grammar file.
 Lemon is able to generate a list of all terminals and nonterminals
 by examining the grammar rules, and it can always distinguish a
@@ -345,8 +319,7 @@ The list of terminals and nonterminals on the right-hand side of the
 rule can be empty.
 Rules can occur in any order, except that the left-hand side of the
 first rule is assumed to be the start symbol for the grammar (unless
-specified otherwise using the <tt><a href='#start_symbol'>%start_symbol</a></tt>
-directive described below.)
+specified otherwise using the <tt>%start</tt> directive described below.)
 A typical sequence of grammar rules might look something like this:
 <pre>
  expr ::= expr PLUS expr.
@@ -389,7 +362,7 @@ names to each symbol in a grammar rule and then using those symbolic
 names in the action.
 In yacc or bison, one would write this:
 <pre>
-  expr -&gt; expr PLUS expr  { $$ = $1 + $3; };
+  expr -> expr PLUS expr  { $$ = $1 + $3; };
 </pre>
 But in Lemon, the same rule becomes the following:
 <pre>
@@ -429,14 +402,14 @@ of the shift, and a reduce-reduce conflict is resolved by reducing
 whichever rule comes first in the grammar file.</p>

 <p>Just like in
-yacc and bison, Lemon allows a measure of control
-over the resolution of parsing conflicts using precedence rules.
+yacc and bison, Lemon allows a measure of control 
+over the resolution of paring conflicts using precedence rules.
 A precedence value can be assigned to any terminal symbol
-using the
-<tt><a href='#pleft'>%left</a></tt>,
-<tt><a href='#pright'>%right</a></tt> or
-<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives.  Terminal symbols
-mentioned in earlier directives have a lower precedence than
+using the 
+<a href='#pleft'>%left</a>,
+<a href='#pright'>%right</a> or
+<a href='#pnonassoc'>%nonassoc</a> directives.  Terminal symbols
+mentioned in earlier directives have a lower precedence that
 terminal symbols mentioned in later directives.  For example:</p>

 <p><pre>
@@ -488,7 +461,7 @@ is an error.</p>
 The precedence of a grammar rule is equal to the precedence of the
 left-most terminal symbol in the rule for which a precedence is
 defined.  This is normally what you want, but in those cases where
-you want the precedence of a grammar rule to be something different,
+you want to precedence of a grammar rule to be something different,
 you can specify an alternative precedence symbol by putting the
 symbol in square braces after the period at the end of the rule and
 before any C-code.  For example:</p>
@@ -512,29 +485,29 @@ as follows:
 <li> If the precedence of the token to be shifted is greater than
     the precedence of the rule to reduce, then resolve in favor
     of the shift.  No parsing conflict is reported.
-<li> If the precedence of the token to be shifted is less than the
+<li> If the precedence of the token it be shifted is less than the
     precedence of the rule to reduce, then resolve in favor of the
     reduce action.  No parsing conflict is reported.
 <li> If the precedences are the same and the shift token is
     right-associative, then resolve in favor of the shift.
     No parsing conflict is reported.
-<li> If the precedences are the same and the shift token is
+<li> If the precedences are the same the shift token is
     left-associative, then resolve in favor of the reduce.
     No parsing conflict is reported.
-<li> Otherwise, resolve the conflict by doing the shift, and
-     report a parsing conflict.
+<li> Otherwise, resolve the conflict by doing the shift and
+     report the parsing conflict.
 </ul>
 Reduce-reduce conflicts are resolved this way:
 <ul>
-<li> If either reduce rule
+<li> If either reduce rule 
     lacks precedence information, then resolve in favor of the
-     rule that appears first in the grammar, and report a parsing
+     rule that appears first in the grammar and report a parsing
     conflict.
-<li> If both rules have precedence and the precedence is different,
+<li> If both rules have precedence and the precedence is different
     then resolve the dispute in favor of the rule with the highest
-     precedence, and do not report a conflict.
+     precedence and do not report a conflict.
 <li> Otherwise, resolve the conflict by reducing by the rule that
-     appears first in the grammar, and report a parsing conflict.
+     appears first in the grammar and report a parsing conflict.
 </ul>

 <h3>Special Directives</h3>
@@ -543,40 +516,40 @@ Reduce-reduce conflicts are resolved this way:
 directives.  We've described all the grammar rules, so now we'll
 talk about the special directives.</p>

-<p>Directives in Lemon can occur in any order.  You can put them before
-the grammar rules, or after the grammar rules, or in the midst of the
+<p>Directives in lemon can occur in any order.  You can put them before
+the grammar rules, or after the grammar rules, or in the mist of the
 grammar rules.  It doesn't matter.  The relative order of
 directives used to assign precedence to terminals is important, but
 other than that, the order of directives in Lemon is arbitrary.</p>

 <p>Lemon supports the following special directives:
 <ul>
-<li><tt><a href='#pcode'>%code</a></tt>
-<li><tt><a href='#default_destructor'>%default_destructor</a></tt>
-<li><tt><a href='#default_type'>%default_type</a></tt>
-<li><tt><a href='#destructor'>%destructor</a></tt>
-<li><tt><a href='#pifdef'>%endif</a></tt>
-<li><tt><a href='#extraarg'>%extra_argument</a></tt>
-<li><tt><a href='#pfallback'>%fallback</a></tt>
-<li><tt><a href='#pifdef'>%ifdef</a></tt>
-<li><tt><a href='#pifdef'>%ifndef</a></tt>
-<li><tt><a href='#pinclude'>%include</a></tt>
-<li><tt><a href='#pleft'>%left</a></tt>
-<li><tt><a href='#pname'>%name</a></tt>
-<li><tt><a href='#pnonassoc'>%nonassoc</a></tt>
-<li><tt><a href='#parse_accept'>%parse_accept</a></tt>
-<li><tt><a href='#parse_failure'>%parse_failure</a></tt>
-<li><tt><a href='#pright'>%right</a></tt>
-<li><tt><a href='#stack_overflow'>%stack_overflow</a></tt>
-<li><tt><a href='#stack_size'>%stack_size</a></tt>
-<li><tt><a href='#start_symbol'>%start_symbol</a></tt>
-<li><tt><a href='#syntax_error'>%syntax_error</a></tt>
-<li><tt><a href='#token_class'>%token_class</a></tt>
-<li><tt><a href='#token_destructor'>%token_destructor</a></tt>
-<li><tt><a href='#token_prefix'>%token_prefix</a></tt>
-<li><tt><a href='#token_type'>%token_type</a></tt>
-<li><tt><a href='#ptype'>%type</a></tt>
-<li><tt><a href='#pwildcard'>%wildcard</a></tt>
+<li><tt>%code</tt>
+<li><tt>%default_destructor</tt>
+<li><tt>%default_type</tt>
+<li><tt>%destructor</tt>
+<li><tt>%endif</tt>
+<li><tt>%extra_argument</tt>
+<li><tt>%fallback</tt>
+<li><tt>%ifdef</tt>
+<li><tt>%ifndef</tt>
+<li><tt>%include</tt>
+<li><tt>%left</tt>
+<li><tt>%name</tt>
+<li><tt>%nonassoc</tt>
+<li><tt>%parse_accept</tt>
+<li><tt>%parse_failure </tt>
+<li><tt>%right</tt>
+<li><tt>%stack_overflow</tt>
+<li><tt>%stack_size</tt>
+<li><tt>%start_symbol</tt>
+<li><tt>%syntax_error</tt>
+<li><tt>%token_class</tt>
+<li><tt>%token_destructor</tt>
+<li><tt>%token_prefix</tt>
+<li><tt>%token_type</tt>
+<li><tt>%type</tt>
+<li><tt>%wildcard</tt>
 </ul>
 Each of these directives will be described separately in the
 following sections:</p>
@@ -584,42 +557,43 @@ following sections:</p>
 <a name='pcode'></a>
 <h4>The <tt>%code</tt> directive</h4>

-<p>The <tt>%code</tt> directive is used to specify additional C code that
+<p>The %code directive is used to specify addition C code that
 is added to the end of the main output file.  This is similar to
-the <tt><a href='#pinclude'>%include</a></tt> directive except that
-<tt>%include</tt> is inserted at the beginning of the main output file.</p>
+the <a href='#pinclude'>%include</a> directive except that %include
+is inserted at the beginning of the main output file.</p>

-<p><tt>%code</tt> is typically used to include some action routines or perhaps
-a tokenizer or even the "main()" function
+<p>%code is typically used to include some action routines or perhaps
+a tokenizer or even the "main()" function 
 as part of the output file.</p>

 <a name='default_destructor'></a>
 <h4>The <tt>%default_destructor</tt> directive</h4>

-<p>The <tt>%default_destructor</tt> directive specifies a destructor to
+<p>The %default_destructor directive specifies a destructor to 
 use for non-terminals that do not have their own destructor
-specified by a separate <tt>%destructor</tt> directive.  See the documentation
-on the <tt><a name='#destructor'>%destructor</a></tt> directive below for
+specified by a separate %destructor directive.  See the documentation
+on the <a name='#destructor'>%destructor</a> directive below for
 additional information.</p>

-<p>In some grammars, many different non-terminal symbols have the
-same data type and hence the same destructor.  This directive is
-a convenient way to specify the same destructor for all those
+<p>In some grammers, many different non-terminal symbols have the
+same datatype and hence the same destructor.  This directive is
+a convenience way to specify the same destructor for all those
 non-terminals using a single statement.</p>

 <a name='default_type'></a>
 <h4>The <tt>%default_type</tt> directive</h4>

-<p>The <tt>%default_type</tt> directive specifies the data type of non-terminal
-symbols that do not have their own data type defined using a separate
-<tt><a href='#ptype'>%type</a></tt> directive.</p>
+<p>The %default_type directive specifies the datatype of non-terminal
+symbols that do no have their own datatype defined using a separate
+<a href='#ptype'>%type</a> directive.  
+</p>

 <a name='destructor'></a>
 <h4>The <tt>%destructor</tt> directive</h4>

-<p>The <tt>%destructor</tt> directive is used to specify a destructor for
+<p>The %destructor directive is used to specify a destructor for
 a non-terminal symbol.
-(See also the <tt><a href='#token_destructor'>%token_destructor</a></tt>
+(See also the <a href='#token_destructor'>%token_destructor</a>
 directive which is used to specify a destructor for terminal symbols.)</p>

 <p>A non-terminal's destructor is called to dispose of the
@@ -641,7 +615,7 @@ or other resources held by that non-terminal.</p>
   %destructor nt { free($$); }
   nt(A) ::= ID NUM.   { A = malloc( 100 ); }
 </pre>
-This example is a bit contrived, but it serves to illustrate how
+This example is a bit contrived but it serves to illustrate how
 destructors work.  The example shows a non-terminal named
 "nt" that holds values of type "void*".  When the rule for
 an "nt" reduces, it sets the value of the non-terminal to
@@ -657,17 +631,17 @@ stack, unless the non-terminal is used in a C-code action.  If
 the non-terminal is used by C-code, then it is assumed that the
 C-code will take care of destroying it.
 More commonly, the value is used to build some
-larger structure, and we don't want to destroy it, which is why
+larger structure and we don't want to destroy it, which is why
 the destructor is not called in this circumstance.</p>

 <p>Destructors help avoid memory leaks by automatically freeing
 allocated objects when they go out of scope.
 To do the same using yacc or bison is much more difficult.</p>

-<a name='extraarg'></a>
+<a name="extraarg"></a>
 <h4>The <tt>%extra_argument</tt> directive</h4>

-The <tt>%extra_argument</tt> directive instructs Lemon to add a 4th parameter
+The %extra_argument directive instructs Lemon to add a 4th parameter
 to the parameter list of the Parse() function it generates.  Lemon
 doesn't do anything itself with this extra argument, but it does
 make the argument available to C-code action routines, destructors,
@@ -682,91 +656,64 @@ of type "MyStruct*" and all action routines will have access to
 a variable named "pAbc" that is the value of the 4th parameter
 in the most recent call to Parse().</p>

-<p>The <tt>%extra_context</tt> directive works the same except that it
-is passed in on the ParseAlloc() or ParseInit() routines instead of
-on Parse().
-
-<a name='extractx'></a>
-<h4>The <tt>%extra_context</tt> directive</h4>
-
-The <tt>%extra_context</tt> directive instructs Lemon to add a 2nd parameter
-to the parameter list of the ParseAlloc() and ParseInif() functions.  Lemon
-doesn't do anything itself with these extra argument, but it does
-store the value make it available to C-code action routines, destructors,
-and so forth.  For example, if the grammar file contains:</p>
-
-<p><pre>
-    %extra_context { MyStruct *pAbc }
-</pre></p>
-
-<p>Then the ParseAlloc() and ParseInit() functions will have an 2nd parameter
-of type "MyStruct*" and all action routines will have access to
-a variable named "pAbc" that is the value of that 2nd parameter.</p>
-
-<p>The <tt>%extra_argument</tt> directive works the same except that it
-is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().
-
 <a name='pfallback'></a>
 <h4>The <tt>%fallback</tt> directive</h4>

-<p>The <tt>%fallback</tt> directive specifies an alternative meaning for one
+<p>The %fallback directive specifies an alternative meaning for one
 or more tokens.  The alternative meaning is tried if the original token
-would have generated a syntax error.</p>
+would have generated a syntax error.

-<p>The <tt>%fallback</tt> directive was added to support robust parsing of SQL
-syntax in <a href='https://www.sqlite.org/'>SQLite</a>.
+<p>The %fallback directive was added to support robust parsing of SQL
+syntax in <a href="https://www.sqlite.org/">SQLite</a>.
 The SQL language contains a large assortment of keywords, each of which
 appears as a different token to the language parser.  SQL contains so
-many keywords that it can be difficult for programmers to keep up with
+many keywords, that it can be difficult for programmers to keep up with
 them all.  Programmers will, therefore, sometimes mistakenly use an
-obscure language keyword for an identifier.  The <tt>%fallback</tt> directive
+obscure language keyword for an identifier.  The %fallback directive
 provides a mechanism to tell the parser:  "If you are unable to parse
-this keyword, try treating it as an identifier instead."</p>
+this keyword, try treating it as an identifier instead."

-<p>The syntax of <tt>%fallback</tt> is as follows:
+<p>The syntax of %fallback is as follows:

 <blockquote>
-<tt>%fallback</tt> <i>ID</i> <i>TOKEN...</i> <b>.</b>
-</blockquote></p>
+<tt>%fallback</tt>  <i>ID</i> <i>TOKEN...</i> <b>.</b>
+</blockquote>

-<p>In words, the <tt>%fallback</tt> directive is followed by a list of token
-names terminated by a period.
-The first token name is the fallback token &mdash; the
+<p>In words, the %fallback directive is followed by a list of token names
+terminated by a period.  The first token name is the fallback token - the
 token to which all the other tokens fall back to.  The second and subsequent
 arguments are tokens which fall back to the token identified by the first
-argument.</p>
+argument.

 <a name='pifdef'></a>
-<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives</h4>
+<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4>

-<p>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives
-are similar to #ifdef, #ifndef, and #endif in the C-preprocessor,
-just not as general.
+<p>The %ifdef, %ifndef, and %endif directives are similar to
+#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general.
 Each of these directives must begin at the left margin.  No whitespace
-is allowed between the "%" and the directive name.</p>
+is allowed between the "%" and the directive name.

-<p>Grammar text in between "<tt>%ifdef MACRO</tt>" and the next nested
-"<tt>%endif</tt>" is
+<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is
 ignored unless the "-DMACRO" command-line option is used.  Grammar text
-betwen "<tt>%ifndef MACRO</tt>" and the next nested "<tt>%endif</tt>" is
-included except when the "-DMACRO" command-line option is used.</p>
+betwen "%ifndef MACRO" and the next nested "%endif" is included except when
+the "-DMACRO" command-line option is used.

-<p>Note that the argument to <tt>%ifdef</tt> and <tt>%ifndef</tt> must
-be a single preprocessor symbol name, not a general expression.
-There is no "<tt>%else</tt>" directive.</p>
+<p>Note that the argument to %ifdef and %ifndef must be a single 
+preprocessor symbol name, not a general expression.  There is no "%else"
+directive.


 <a name='pinclude'></a>
 <h4>The <tt>%include</tt> directive</h4>

-<p>The <tt>%include</tt> directive specifies C code that is included at the
-top of the generated parser.  You can include any text you want &mdash;
+<p>The %include directive specifies C code that is included at the
+top of the generated parser.  You can include any text you want --
 the Lemon parser generator copies it blindly.  If you have multiple
-<tt>%include</tt> directives in your grammar file, their values are concatenated
-so that all <tt>%include</tt> code ultimately appears near the top of the
-generated parser, in the same order as it appeared in the grammar.</p>
+%include directives in your grammar file, their values are concatenated
+so that all %include code ultimately appears near the top of the
+generated parser, in the same order as it appeared in the grammer.</p>

-<p>The <tt>%include</tt> directive is very handy for getting some extra #include
+<p>The %include directive is very handy for getting some extra #include
 preprocessor statements at the beginning of the generated parser.
 For example:</p>

@@ -775,22 +722,17 @@ For example:</p>
 </pre></p>

 <p>This might be needed, for example, if some of the C actions in the
-grammar call functions that are prototyped in unistd.h.</p>
-
-<p>Use the <tt><a href="#pcode">%code</a></tt> directive to add code to
-the end of the generated parser.</p>
+grammar call functions that are prototyed in unistd.h.</p>

 <a name='pleft'></a>
 <h4>The <tt>%left</tt> directive</h4>

-The <tt>%left</tt> directive is used (along with the
-<tt><a href='#pright'>%right</a></tt> and
-<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives) to declare
-precedences of terminal symbols.
-Every terminal symbol whose name appears after
-a <tt>%left</tt> directive but before the next period (".") is
+The %left directive is used (along with the <a href='#pright'>%right</a> and
+<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of 
+terminal symbols.  Every terminal symbol whose name appears after
+a %left directive but before the next period (".") is
 given the same left-associative precedence value.  Subsequent
-<tt>%left</tt> directives have higher precedence.  For example:</p>
+%left directives have higher precedence.  For example:</p>

 <p><pre>
   %left AND.
@@ -801,21 +743,20 @@ given the same left-associative precedence value.  Subsequent
   %right EXP NOT.
 </pre></p>

-<p>Note the period that terminates each <tt>%left</tt>,
-<tt>%right</tt> or <tt>%nonassoc</tt>
+<p>Note the period that terminates each %left, %right or %nonassoc
 directive.</p>

 <p>LALR(1) grammars can get into a situation where they require
 a large amount of stack space if you make heavy use or right-associative
-operators.  For this reason, it is recommended that you use <tt>%left</tt>
-rather than <tt>%right</tt> whenever possible.</p>
+operators.  For this reason, it is recommended that you use %left
+rather than %right whenever possible.</p>

 <a name='pname'></a>
 <h4>The <tt>%name</tt> directive</h4>

 <p>By default, the functions generated by Lemon all begin with the
 five-character string "Parse".  You can change this string to something
-different using the <tt>%name</tt> directive.  For instance:</p>
+different using the %name directive.  For instance:</p>

 <p><pre>
   %name Abcde
@@ -829,22 +770,22 @@ functions named
 <li> AbcdeTrace(), and
 <li> Abcde().
 </ul>
-The <tt>%name</tt> directive allows you to generate two or more different
-parsers and link them all into the same executable.</p>
+The %name directive allows you to generator two or more different
+parsers and link them all into the same executable.
+</p>

 <a name='pnonassoc'></a>
 <h4>The <tt>%nonassoc</tt> directive</h4>

 <p>This directive is used to assign non-associative precedence to
-one or more terminal symbols.  See the section on
+one or more terminal symbols.  See the section on 
 <a href='#precrules'>precedence rules</a>
-or on the <tt><a href='#pleft'>%left</a></tt> directive
-for additional information.</p>
+or on the <a href='#pleft'>%left</a> directive for additional information.</p>

 <a name='parse_accept'></a>
 <h4>The <tt>%parse_accept</tt> directive</h4>

-<p>The <tt>%parse_accept</tt> directive specifies a block of C code that is
+<p>The %parse_accept directive specifies a block of C code that is
 executed whenever the parser accepts its input string.  To "accept"
 an input string means that the parser was able to process all tokens
 without error.</p>
@@ -860,7 +801,7 @@ without error.</p>
 <a name='parse_failure'></a>
 <h4>The <tt>%parse_failure</tt> directive</h4>

-<p>The <tt>%parse_failure</tt> directive specifies a block of C code that
+<p>The %parse_failure directive specifies a block of C code that
 is executed whenever the parser fails complete.  This code is not
 executed until the parser has tried and failed to resolve an input
 error using is usual error recovery strategy.  The routine is
@@ -876,14 +817,14 @@ only invoked when parsing is unable to continue.</p>
 <h4>The <tt>%right</tt> directive</h4>

 <p>This directive is used to assign right-associative precedence to
-one or more terminal symbols.  See the section on
+one or more terminal symbols.  See the section on 
 <a href='#precrules'>precedence rules</a>
 or on the <a href='#pleft'>%left</a> directive for additional information.</p>

 <a name='stack_overflow'></a>
 <h4>The <tt>%stack_overflow</tt> directive</h4>

-<p>The <tt>%stack_overflow</tt> directive specifies a block of C code that
+<p>The %stack_overflow directive specifies a block of C code that
 is executed if the parser's internal stack ever overflows.  Typically
 this just prints an error message.  After a stack overflow, the parser
 will be unable to continue and must be reset.</p>
@@ -896,7 +837,7 @@ will be unable to continue and must be reset.</p>

 <p>You can help prevent parser stack overflows by avoiding the use
 of right recursion and right-precedence operators in your grammar.
-Use left recursion and and left-precedence operators instead to
+Use left recursion and and left-precedence operators instead, to
 encourage rules to reduce sooner and keep the stack size down.
 For example, do rules like this:
 <pre>
@@ -907,7 +848,7 @@ Not like this:
 <pre>
   list ::= element list.      // right-recursion.  Bad!
   list ::= .
-</pre></p>
+</pre>

 <a name='stack_size'></a>
 <h4>The <tt>%stack_size</tt> directive</h4>
@@ -915,7 +856,7 @@ Not like this:
 <p>If stack overflow is a problem and you can't resolve the trouble
 by using left-recursion, then you might want to increase the size
 of the parser's stack using this directive.  Put an positive integer
-after the <tt>%stack_size</tt> directive and Lemon will generate a parse
+after the %stack_size directive and Lemon will generate a parse
 with a stack of the requested size.  The default value is 100.</p>

 <p><pre>
@@ -925,40 +866,25 @@ with a stack of the requested size.  The default value is 100.</p>
 <a name='start_symbol'></a>
 <h4>The <tt>%start_symbol</tt> directive</h4>

-<p>By default, the start symbol for the grammar that Lemon generates
+<p>By default, the start-symbol for the grammar that Lemon generates
 is the first non-terminal that appears in the grammar file.  But you
-can choose a different start symbol using the
-<tt>%start_symbol</tt> directive.</p>
+can choose a different start-symbol using the %start_symbol directive.</p>

 <p><pre>
   %start_symbol  prog
 </pre></p>

-<a name='syntax_error'></a>
-<h4>The <tt>%syntax_error</tt> directive</h4>
-
-<p>See <a href='#error_processing'>Error Processing</a>.</p>
-
-<a name='token_class'></a>
-<h4>The <tt>%token_class</tt> directive</h4>
-
-<p>Undocumented.  Appears to be related to the MULTITERMINAL concept.
-<a href='http://sqlite.org/src/fdiff?v1=796930d5fc2036c7&v2=624b24c5dc048e09&sbs=0'>Implementation</a>.</p>
-
 <a name='token_destructor'></a>
 <h4>The <tt>%token_destructor</tt> directive</h4>

-<p>The <tt>%destructor</tt> directive assigns a destructor to a non-terminal
-symbol.  (See the description of the
-<tt><a href='%destructor'>%destructor</a></tt> directive above.)
-The <tt>%token_destructor</tt> directive does the same thing
-for all terminal symbols.</p>
+<p>The %destructor directive assigns a destructor to a non-terminal
+symbol.  (See the description of the %destructor directive above.)
+This directive does the same thing for all terminal symbols.</p>

 <p>Unlike non-terminal symbols which may each have a different data type
 for their values, terminals all use the same data type (defined by
-the <tt><a href='#token_type'>%token_type</a></tt> directive)
-and so they use a common destructor.
-Other than that, the token destructor works just like the non-terminal
+the %token_type directive) and so they use a common destructor.  Other
+than that, the token destructor works just like the non-terminal
 destructors.</p>

 <a name='token_prefix'></a>
@@ -967,9 +893,8 @@ destructors.</p>
 <p>Lemon generates #defines that assign small integer constants
 to each terminal symbol in the grammar.  If desired, Lemon will
 add a prefix specified by this directive
-to each of the #defines it generates.</p>
-
-<p>So if the default output of Lemon looked like this:
+to each of the #defines it generates.
+So if the default output of Lemon looked like this:
 <pre>
    #define AND              1
    #define MINUS            2
@@ -986,7 +911,7 @@ to cause Lemon to produce these symbols instead:
    #define TOKEN_MINUS      2
    #define TOKEN_OR         3
    #define TOKEN_PLUS       4
-</pre></p>
+</pre>

 <a name='token_type'></a><a name='ptype'></a>
 <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
@@ -996,7 +921,7 @@ on the parser's stack associated with terminal and non-terminal
 symbols.  The values of all terminal symbols must be of the same
 type.  This turns out to be the same data type as the 3rd parameter
 to the Parse() function generated by Lemon.  Typically, you will
-make the value of a terminal symbol be a pointer to some kind of
+make the value of a terminal symbol by a pointer to some kind of
 token structure.  Like this:</p>

 <p><pre>
@@ -1007,7 +932,7 @@ token structure.  Like this:</p>
 is "void*".</p>

 <p>Non-terminal symbols can each have their own data types.  Typically
-the data type of a non-terminal is a pointer to the root of a parse tree
+the data type  of a non-terminal is a pointer to the root of a parse-tree
 structure that contains all information about that non-terminal.
 For example:</p>

@@ -1028,15 +953,14 @@ and able to pay that price, fine.  You just need to know.</p>
 <a name='pwildcard'></a>
 <h4>The <tt>%wildcard</tt> directive</h4>

-<p>The <tt>%wildcard</tt> directive is followed by a single token name and a
-period.  This directive specifies that the identified token should
-match any input token.</p>
+<p>The %wildcard directive is followed by a single token name and a
+period.  This directive specifies that the identified token should 
+match any input token.

 <p>When the generated parser has the choice of matching an input against
 the wildcard token and some other token, the other token is always used.
-The wildcard token is only matched if there are no alternatives.</p>
+The wildcard token is only matched if there are no other alternatives.

-<a name='error_processing'></a>
 <h3>Error Processing</h3>

 <p>After extensive experimentation over several years, it has been
@@ -1044,20 +968,19 @@ discovered that the error recovery strategy used by yacc is about
 as good as it gets.  And so that is what Lemon uses.</p>

 <p>When a Lemon-generated parser encounters a syntax error, it
-first invokes the code specified by the <tt>%syntax_error</tt> directive, if
+first invokes the code specified by the %syntax_error directive, if
 any.  It then enters its error recovery strategy.  The error recovery
 strategy is to begin popping the parsers stack until it enters a
 state where it is permitted to shift a special non-terminal symbol
 named "error".  It then shifts this non-terminal and continues
-parsing.  The <tt>%syntax_error</tt> routine will not be called again
+parsing.  But the %syntax_error routine will not be called again
 until at least three new tokens have been successfully shifted.</p>

 <p>If the parser pops its stack until the stack is empty, and it still
-is unable to shift the error symbol, then the
-<tt><a href='#parse_failure'>%parse_failure</a></tt> routine
+is unable to shift the error symbol, then the %parse_failed routine
 is invoked and the parser resets itself to its start state, ready
 to begin parsing a new file.  This is what will happen at the very
-first syntax error, of course, if there are no instances of the
+first syntax error, of course, if there are no instances of the 
 "error" non-terminal in your grammar.</p>

 </body>
@@ -1,142 +0,0 @@
-# The new-security-options branch
-
-## The problem that the [new-security-options](/timeline?r=new-security-options) branch tries to solve
-
-An attacker might modify the schema of an SQLite database by adding
-structures that cause code to run when some other application opens and
-reads the database.  For example, the attacker might replace a table
-definition with a view.  Or the attacker might add triggers to tables
-or views, or add new CHECK constraints or generated columns or indexes
-with expressions in the index list or in the WHERE clause.  If the
-added features invoke SQL functions or virtual tables with side effects,
-that might cause harm to the system if run by a high-privilege victim.
-Or, the added features might exfiltrate information if the database is
-read by a high-privilege victim.
-
-The changes in this branch strive to make it easier for high-privilege
-applications to safely read SQLite database files that might have been
-maliciously corrupted by an attacker.
-
-## Overview of changes in [new-security-options](/timeline?r=new-security-options)
-
-The basic idea is to tag every SQL function and virtual table with one
-of three risk levels:
-
-  1.  Innocuous
-  2.  Normal
-  3.  Direct-Only
-
-Innocuous functions/vtabs are safe and can be used at any time.
-Direct-only elements, in contrast, might have cause side-effects and
-should only be used from top-level SQL, not from within triggers or views nor
-in elements of the schema such as CHECK constraint, DEFAULT values, 
-generated columns, index expressions, or in the WHERE clause of a 
-partial index that are potentially under the control of an attacker.
-Normal elements behave like Innocuous if TRUSTED\_SCHEMA=on
-and behave like direct-only if TRUSTED\_SCHEMA=off.
-
-Application-defined functions and virtual tables go in as Normal unless
-the application takes deliberate steps to change the risk level.
-
-For backwards compatibility, the default is TRUSTED\_SCHEMA=on.  Documentation
-will be updated to recommend applications turn TRUSTED\_SCHEMA to off.
-
-An innocuous function or virtual table is one that can only read content
-from the database file in which it resides, and can only alter the database
-in which it resides.  Most SQL functions are innocuous.  For example, there
-is no harm in an attacker running the abs() function.
-
-Direct-only elements that have side-effects that go outside the database file
-in which it lives, or return information from outside of the database file.
-Examples of direct-only elements include:
-
-  1.  The fts3\_tokenizer() function
-  2.  The writefile() function
-  3.  The readfile() function
-  4.  The zipvfs virtual table
-  5.  The csv virtual table
-
-We do not want an attacker to be able to add these kinds of things to
-the database schema and possibly trick a high-privilege application 
-from performing any of these actions.  Therefore, functions and vtabs
-with side-effects are marked as Direct-Only.
-
-Legacy applications might add other risky functions or vtabs.  Those will
-go in as "Normal" by default.  For optimal security, we want those risky
-app-defined functions and vtabs to be direct-only, but making that the
-default might break some legacy applications.  Hence, all app-defined
-functions and vtabs go in as Normal, but the application can switch them
-over to "Direct-Only" behavior using a single pragma.
-
-The restrictions on the use of functions and virtual tables do not apply
-to TEMP.  A TEMP VIEW or a TEMP TRIGGER can use any valid SQL function
-or virtual table.  The idea is that TEMP views and triggers must be
-directly created by the application and are thus under the control of the
-application.  TEMP views and triggers cannot be created by an attacker who
-corrupts the schema of a persistent database file.  Hence TEMP views and
-triggers are safe.
-
-## Specific changes
-
-  1.  New sqlite3\_db\_config() option SQLITE\_DBCONFIG\_TRUSTED\_SCHEMA for
-      turning TRUSTED\_SCHEMA on and off.  It defaults to ON.
-
-  2.  Compile-time option -DSQLITE\_TRUSTED\_SCHEMA=0 causes the default
-      TRUSTED\_SCHEMA setting to be off.
-
-  3.  New pragma "PRAGMA trusted\_schema=(ON\|OFF);".  This provides access
-      to the TRUSTED_SCHEMA setting for application coded using scripting
-      languages or other secondary languages where they are unable to make
-      calls to sqlite3\_db\_config().
-
-  4.  New options for the "enc" parameter to sqlite3\_create\_function() and
-      its kin:
-      <ol type="a">
-      <li>  _SQLITE\_INNOCUOUS_  &rarr; tags the new functions as Innocuous
-      <li>  _SQLITE\_DIRECTONLY_ &rarr; tags the new functions as Direct-Only
-      </ol>
-
-  5.  New options to sqlite3\_vtab\_config():
-      <ol type="a">
-      <li>  _SQLITE\_VTAB\_INNOCUOUS_   &rarr; tags the vtab as Innocuous
-      <li>  _SQLITE\_VTAB\_DIRECTONLY_  &rarr; tags the vtab as Direct-Only
-      </ol>
-
-  6.  Change many of the functions and virtual tables in the SQLite source
-      tree to use one of the tags above.
-
-  7.  Enhanced PRAGMA function\_list and virtual-table "pragma\_function\_list"
-      with additional columns.  The columns now are:
-      <ul>
-      <li> _name_      &rarr;  Name of the function
-      <li> _builtin_   &rarr;  1 for built-in functions.  0 otherwise.
-      <li> _type_      &rarr;  's'=Scalar, 'a'=Aggregate, 'w'=Window
-      <li> _enc_       &rarr;  'utf8', 'utf16le', or 'utf16be'
-      <li> _narg_      &rarr;  number of argument
-      <li> _flags_     &rarr;  Bitmask of SQLITE\_INNOCUOUS, SQLITE\_DIRECTONLY,
-                               SQLITE\_DETERMINISTIC, SQLITE\_SUBTYPE, and
-                               SQLITE\_FUNC\_INTERNAL flags.
-      </ul>
-      <p>The last four columns are new.
-
-  8.  The function\_list PRAGMA now also shows all entries for each function.
-      So, for example, if a function can take either 2 or 3 arguments,
-      there are separate rows for the 2-argument and 3-argument versions of
-      the function.
-
-## Additional Notes
-
-The function_list enhancements allow the application to query the set
-of SQL functions that meet various criteria.  For example, to see all
-SQL functions that are never allowed to be used in the schema or in
-trigger or views:
-
-~~~
-    SELECT DISTINCT name FROM pragma_function_list
-     WHERE (flags & 0x80000)!=0
-     ORDER BY name;
-~~~
-
-Doing the same is not possible for virtual tables, as a virtual table
-might be Innocuous, Normal, or Direct-Only depending on the arguments
-passed into the xConnect method.
@@ -1,8 +0,0 @@
-## Loadable Extensions
-
-Various [loadable extensions](https://www.sqlite.org/loadext.html) for
-SQLite are found in subfolders.
-
-Most subfolders are dedicated to a single loadable extension (for
-example FTS5, or RTREE).  But the misc/ subfolder contains a collection
-of smaller single-file extensions.
@@ -0,0 +1,2 @@
+Version loadable extensions to SQLite are found in subfolders
+of this folder.
@@ -1,83 +0,0 @@
-## SQLite Expert Extension
-
-This folder contains code for a simple system to propose useful indexes
-given a database and a set of SQL queries. It works as follows:
-
-  1. The user database schema is copied to a temporary database.
-
-  1. All SQL queries are prepared against the temporary database.
-     Information regarding the WHERE and ORDER BY clauses, and other query
-     features that affect index selection are recorded.
-
-  1. The information gathered in step 2 is used to create candidate 
-     indexes - indexes that the planner might have made use of in the previous
-     step, had they been available.
-
-  1. A subset of the data in the user database is used to generate statistics
-     for all existing indexes and the candidate indexes generated in step 3
-     above.
-
-  1. The SQL queries are prepared a second time. If the planner uses any
-     of the indexes created in step 3, they are recommended to the user.
-
-# C API
-
-The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
-as follows:
-
-  1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**.
-     A database handle opened by the user is passed as an argument.
-
-  1. The sqlite3expert object is configured with one or more SQL statements
-     by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
-     specify a single SQL statement, or multiple statements separated by
-     semi-colons.
-  
-  1. Optionally, the **sqlite3\_expert\_config()** API may be used to 
-     configure the size of the data subset used to generate index statistics.
-     Using a smaller subset of the data can speed up the analysis.
-
-  1. **sqlite3\_expert\_analyze()** is called to run the analysis.
-
-  1. One or more calls are made to **sqlite3\_expert\_report()** to extract
-     components of the results of the analysis.
-
-  1. **sqlite3\_expert\_destroy()** is called to free all resources.
-
-Refer to comments in sqlite3expert.h for further details.
-
-# sqlite3_expert application
-
-The file "expert.c" contains the code for a command line application that
-uses the API described above. It can be compiled with (for example):
-
-<pre>
-  gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
-</pre>
-
-Assuming the database is named "test.db", it can then be run to analyze a
-single query:
-
-<pre>
-  ./sqlite3_expert -sql &lt;sql-query&gt; test.db
-</pre>
-
-Or an entire text file worth of queries with:
-
-<pre>
-  ./sqlite3_expert -file &lt;text-file&gt; test.db
-</pre>
-
-By default, sqlite3\_expert generates index statistics using all the data in
-the user database. For a large database, this may be prohibitively time
-consuming. The "-sample" option may be used to configure sqlite3\_expert to
-generate statistics based on an integer percentage of the user database as
-follows:
-
-<pre>
-  # Generate statistics based on 25% of the user database rows:
-  ./sqlite3_expert -sample 25 -sql &lt;sql-query&gt; test.db
-
-  # Do not generate any statistics at all:
-  ./sqlite3_expert -sample 0 -sql &lt;sql-query&gt; test.db
-</pre>
@@ -1,156 +0,0 @@
-/*
-** 2017 April 07
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-*/
-
-
-#include <sqlite3.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "sqlite3expert.h"
-
-
-static void option_requires_argument(const char *zOpt){
-  fprintf(stderr, "Option requires an argument: %s\n", zOpt);
-  exit(-3);
-}
-
-static int option_integer_arg(const char *zVal){
-  return atoi(zVal);
-}
-
-static void usage(char **argv){
-  fprintf(stderr, "\n");
-  fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
-  fprintf(stderr, "\n");
-  fprintf(stderr, "Options are:\n");
-  fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
-  fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
-  fprintf(stderr, "  -verbose LEVEL (integer verbosity level. default 1)\n");
-  fprintf(stderr, "  -sample PERCENT (percent of db to sample. default 100)\n");
-  exit(-1);
-}
-
-static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
-  FILE *in = fopen(zFile, "rb");
-  long nIn;
-  size_t nRead;
-  char *pBuf;
-  int rc;
-  if( in==0 ){
-    *pzErr = sqlite3_mprintf("failed to open file %s\n", zFile);
-    return SQLITE_ERROR;
-  }
-  fseek(in, 0, SEEK_END);
-  nIn = ftell(in);
-  rewind(in);
-  pBuf = sqlite3_malloc64( nIn+1 );
-  nRead = fread(pBuf, nIn, 1, in);
-  fclose(in);
-  if( nRead!=1 ){
-    sqlite3_free(pBuf);
-    *pzErr = sqlite3_mprintf("failed to read file %s\n", zFile);
-    return SQLITE_ERROR;
-  }
-  pBuf[nIn] = 0;
-  rc = sqlite3_expert_sql(p, pBuf, pzErr);
-  sqlite3_free(pBuf);
-  return rc;
-}
-
-int main(int argc, char **argv){
-  const char *zDb;
-  int rc = 0;
-  char *zErr = 0;
-  int i;
-  int iVerbose = 1;               /* -verbose option */
-
-  sqlite3 *db = 0;
-  sqlite3expert *p = 0;
-
-  if( argc<2 ) usage(argv);
-  zDb = argv[argc-1];
-  if( zDb[0]=='-' ) usage(argv);
-  rc = sqlite3_open(zDb, &db);
-  if( rc!=SQLITE_OK ){
-    fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
-    exit(-2);
-  }
-
-  p = sqlite3_expert_new(db, &zErr);
-  if( p==0 ){
-    fprintf(stderr, "Cannot run analysis: %s\n", zErr);
-    rc = 1;
-  }else{
-    for(i=1; i<(argc-1); i++){
-      char *zArg = argv[i];
-      int nArg;
-      if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
-      nArg = (int)strlen(zArg);
-      if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
-        if( ++i==(argc-1) ) option_requires_argument("-file");
-        rc = readSqlFromFile(p, argv[i], &zErr);
-      }
-
-      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
-        if( ++i==(argc-1) ) option_requires_argument("-sql");
-        rc = sqlite3_expert_sql(p, argv[i], &zErr);
-      }
-
-      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
-        int iSample;
-        if( ++i==(argc-1) ) option_requires_argument("-sample");
-        iSample = option_integer_arg(argv[i]);
-        sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
-      }
-
-      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
-        if( ++i==(argc-1) ) option_requires_argument("-verbose");
-        iVerbose = option_integer_arg(argv[i]);
-      }
-
-      else{
-        usage(argv);
-      }
-    }
-  }
-
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_expert_analyze(p, &zErr);
-  }
-
-  if( rc==SQLITE_OK ){
-    int nQuery = sqlite3_expert_count(p);
-    if( iVerbose>0 ){
-      const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
-      fprintf(stdout, "-- Candidates -------------------------------\n");
-      fprintf(stdout, "%s\n", zCand);
-    }
-    for(i=0; i<nQuery; i++){
-      const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
-      const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
-      const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
-      if( zIdx==0 ) zIdx = "(no new indexes)\n";
-      if( iVerbose>0 ){
-        fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
-        fprintf(stdout, "%s\n\n", zSql);
-      }
-      fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
-    }
-  }else{
-    fprintf(stderr, "Error: %s\n", zErr ? zErr : "?");
-  }
-
-  sqlite3_expert_destroy(p);
-  sqlite3_free(zErr);
-  return rc;
-}
@@ -1,393 +0,0 @@
-# 2009 Nov 11
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# The focus of this file is testing the CLI shell tool. Specifically,
-# the ".recommend" command.
-#
-#
-
-# Test plan:
-#
-#
-if {![info exists testdir]} {
-  set testdir [file join [file dirname [info script]] .. .. test]
-}
-source $testdir/tester.tcl
-set testprefix expert1
-
-if {[info commands sqlite3_expert_new]==""} {
-  finish_test
-  return
-}
-
-set CLI [test_binary_name sqlite3]
-set CMD [test_binary_name sqlite3_expert]
-
-proc squish {txt} {
-  regsub -all {[[:space:]]+} $txt { }
-}
-
-proc do_setup_rec_test {tn setup sql res} {
-  reset_db
-  if {[info exists ::set_main_db_name]} {
-    dbconfig_maindbname_icecube db
-  }
-  db eval $setup
-  uplevel [list do_rec_test $tn $sql $res]
-}
-
-foreach {tn setup} {
-  1 {
-    if {![file executable $CMD]} { continue }
-
-    proc do_rec_test {tn sql res} {
-      set res [squish [string trim $res]]
-      set tst [subst -nocommands { 
-        squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
-      }]
-      uplevel [list do_test $tn $tst $res]
-    }
-  }
-  2 {
-    if {[info commands sqlite3_expert_new]==""} { continue }
-
-    proc do_rec_test {tn sql res} {
-      set expert [sqlite3_expert_new db]
-      $expert sql $sql
-      $expert analyze
-
-      set result [list]
-      for {set i 0} {$i < [$expert count]} {incr i} {
-        set idx [string trim [$expert report $i indexes]]
-        if {$idx==""} {set idx "(no new indexes)"}
-        lappend result $idx
-        lappend result [string trim [$expert report $i plan]]
-      }
-
-      $expert destroy
-
-      set tst [subst -nocommands {set {} [squish [join {$result}]]}]
-      uplevel [list do_test $tn $tst [string trim [squish $res]]]
-    }
-  }
-  3 {
-    if {[info commands sqlite3_expert_new]==""} { continue }
-    set ::set_main_db_name 1
-  }
-  4 {
-    if {![file executable $CLI]} { continue }
-
-    proc do_rec_test {tn sql res} {
-      set res [squish [string trim $res]]
-      set tst [subst -nocommands { 
-        squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
-      }]
-      uplevel [list do_test $tn $tst $res]
-    }
-  }
-} {
-
-  eval $setup
-
-
-do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
-  SELECT * FROM t1
-} {
-  (no new indexes)
-  SCAN TABLE t1
-}
-
-do_setup_rec_test $tn.2 {
-  CREATE TABLE t1(a, b, c);
-} {
-  SELECT * FROM t1 WHERE b>?;
-} {
-  CREATE INDEX t1_idx_00000062 ON t1(b);
-  SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
-}
-
-do_setup_rec_test $tn.3 {
-  CREATE TABLE t1(a, b, c);
-} {
-  SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
-} {
-  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
-  SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
-}
-
-do_setup_rec_test $tn.4 {
-  CREATE TABLE t1(a, b, c);
-} {
-  SELECT a FROM t1 ORDER BY b;
-} {
-  CREATE INDEX t1_idx_00000062 ON t1(b);
-  SCAN TABLE t1 USING INDEX t1_idx_00000062
-}
-
-do_setup_rec_test $tn.5 {
-  CREATE TABLE t1(a, b, c);
-} {
-  SELECT a FROM t1 WHERE a=? ORDER BY b;
-} {
-  CREATE INDEX t1_idx_000123a7 ON t1(a, b);
-  SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
-}
-
-if 0 {
-do_setup_rec_test $tn.6 {
-  CREATE TABLE t1(a, b, c);
-} {
-  SELECT min(a) FROM t1
-} {
-  CREATE INDEX t1_idx_00000061 ON t1(a);
-  SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
-}
-}
-
-do_setup_rec_test $tn.7 {
-  CREATE TABLE t1(a, b, c);
-} {
-  SELECT * FROM t1 ORDER BY a, b, c;
-} {
-  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
-  SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
-}
-
-#do_setup_rec_test $tn.1.8 {
-#  CREATE TABLE t1(a, b, c);
-#} {
-#  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
-#} {
-#  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
-#  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
-#}
-
-do_setup_rec_test $tn.8.1 {
-  CREATE TABLE t1(a COLLATE NOCase, b, c);
-} {
-  SELECT * FROM t1 WHERE a=?
-} {
-  CREATE INDEX t1_idx_00000061 ON t1(a);
-  SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
-}
-do_setup_rec_test $tn.8.2 {
-  CREATE TABLE t1(a, b COLLATE nocase, c);
-} {
-  SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
-} {
-  CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
-  SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
-}
-
-
-# Tables with names that require quotes.
-#
-do_setup_rec_test $tn.9.1 {
-  CREATE TABLE "t t"(a, b, c);
-} {
-  SELECT * FROM "t t" WHERE a=?
-} {
-  CREATE INDEX 't t_idx_00000061' ON 't t'(a);
-  SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
-}
-
-do_setup_rec_test $tn.9.2 {
-  CREATE TABLE "t t"(a, b, c);
-} {
-  SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
-} {
-  CREATE INDEX 't t_idx_00000062' ON 't t'(b);
-  SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
-}
-
-# Columns with names that require quotes.
-#
-do_setup_rec_test $tn.10.1 {
-  CREATE TABLE t3(a, "b b", c);
-} {
-  SELECT * FROM t3 WHERE "b b" = ?
-} {
-  CREATE INDEX t3_idx_00050c52 ON t3('b b');
-  SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
-}
-
-do_setup_rec_test $tn.10.2 {
-  CREATE TABLE t3(a, "b b", c);
-} {
-  SELECT * FROM t3 ORDER BY "b b"
-} {
-  CREATE INDEX t3_idx_00050c52 ON t3('b b');
-  SCAN TABLE t3 USING INDEX t3_idx_00050c52
-}
-
-# Transitive constraints
-#
-do_setup_rec_test $tn.11.1 {
-  CREATE TABLE t5(a, b);
-  CREATE TABLE t6(c, d);
-} {
-  SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
-} {
-  CREATE INDEX t5_idx_000123a7 ON t5(a, b);
-  CREATE INDEX t6_idx_00000063 ON t6(c);
-  SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
-  SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
-}
-
-# OR terms.
-#
-do_setup_rec_test $tn.12.1 {
-  CREATE TABLE t7(a, b);
-} {
-  SELECT * FROM t7 WHERE a=? OR b=?
-} {
-  CREATE INDEX t7_idx_00000062 ON t7(b);
-  CREATE INDEX t7_idx_00000061 ON t7(a);
-  MULTI-INDEX OR
-    INDEX 1
-      SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
-    INDEX 2
-      SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
-}
-
-# rowid terms.
-#
-do_setup_rec_test $tn.13.1 {
-  CREATE TABLE t8(a, b);
-} {
-  SELECT * FROM t8 WHERE rowid=?
-} {
-  (no new indexes)
-  SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
-}
-do_setup_rec_test $tn.13.2 {
-  CREATE TABLE t8(a, b);
-} {
-  SELECT * FROM t8 ORDER BY rowid
-} {
-  (no new indexes)
-  SCAN TABLE t8
-}
-do_setup_rec_test $tn.13.3 {
-  CREATE TABLE t8(a, b);
-} {
-  SELECT * FROM t8 WHERE a=? ORDER BY rowid
-} {
-  CREATE INDEX t8_idx_00000061 ON t8(a); 
-  SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
-}
-
-# Triggers
-#
-do_setup_rec_test $tn.14 {
-  CREATE TABLE t9(a, b, c);
-  CREATE TABLE t10(a, b, c);
-  CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
-    UPDATE t10 SET a=new.a WHERE b = new.b;
-  END;
-} {
-  INSERT INTO t9 VALUES(?, ?, ?);
-} {
-  CREATE INDEX t10_idx_00000062 ON t10(b); 
-  SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
-}
-
-do_setup_rec_test $tn.15 {
-  CREATE TABLE t1(a, b);
-  CREATE TABLE t2(c, d);
-
-  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
-  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
-
-  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
-  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
-} {
-  SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
-} {
-  CREATE INDEX t2_idx_00000064 ON t2(d);
-  SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
-  SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
-}
-
-do_setup_rec_test $tn.16 {
-  CREATE TABLE t1(a, b);
-} {
-  SELECT * FROM t1 WHERE b IS NOT NULL;
-} {
-  (no new indexes)
-  SCAN TABLE t1
-}
-
-}
-
-proc do_candidates_test {tn sql res} {
-  set res [squish [string trim $res]]
-
-  set expert [sqlite3_expert_new db]
-  $expert sql $sql
-  $expert analyze
-
-  set candidates [squish [string trim [$expert report 0 candidates]]]
-  $expert destroy
-
-  uplevel [list do_test $tn [list set {} $candidates] $res]
-}
-
-
-reset_db
-do_execsql_test 5.0 {
-  CREATE TABLE t1(a, b);
-  CREATE TABLE t2(c, d);
-
-  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
-  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
-
-  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
-  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
-}
-do_candidates_test 5.1 {
-  SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
-} {
-  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
-  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
-  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
-  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
-}
-
-do_candidates_test 5.2 {
-  SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=?
-} {
-  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17
-  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
-}
-
-do_execsql_test 5.3 {
-  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
-  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
-  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16
-
-  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
-  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
-  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
-
-  ANALYZE;
-  SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
-} {
-  t1 t1_idx_00000061 {100 50} 
-  t1 t1_idx_00000062 {100 20}
-  t1 t1_idx_000123a7 {100 50 17}
-  t2 t2_idx_00000063 {100 20} 
-  t2 t2_idx_00000064 {100 5} 
-  t2 t2_idx_0001295b {100 20 5}
-}
-
-
-finish_test
@@ -1,168 +0,0 @@
-/*
-** 2017 April 07
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-*/
-#if !defined(SQLITEEXPERT_H)
-#define SQLITEEXPERT_H 1
-#include "sqlite3.h"
-
-typedef struct sqlite3expert sqlite3expert;
-
-/*
-** Create a new sqlite3expert object.
-**
-** If successful, a pointer to the new object is returned and (*pzErr) set
-** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to
-** an English-language error message. In this case it is the responsibility
-** of the caller to eventually free the error message buffer using
-** sqlite3_free().
-*/
-sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
-
-/*
-** Configure an sqlite3expert object.
-**
-** EXPERT_CONFIG_SAMPLE:
-**   By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
-**   each candidate index. This involves scanning and sorting the entire
-**   contents of each user database table once for each candidate index
-**   associated with the table. For large databases, this can be 
-**   prohibitively slow. This option allows the sqlite3expert object to
-**   be configured so that sqlite_stat1 data is instead generated based on a
-**   subset of each table, or so that no sqlite_stat1 data is used at all.
-**
-**   A single integer argument is passed to this option. If the value is less
-**   than or equal to zero, then no sqlite_stat1 data is generated or used by
-**   the analysis - indexes are recommended based on the database schema only.
-**   Or, if the value is 100 or greater, complete sqlite_stat1 data is
-**   generated for each candidate index (this is the default). Finally, if the
-**   value falls between 0 and 100, then it represents the percentage of user
-**   table rows that should be considered when generating sqlite_stat1 data.
-**
-**   Examples:
-**
-**     // Do not generate any sqlite_stat1 data
-**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
-**
-**     // Generate sqlite_stat1 data based on 10% of the rows in each table.
-**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
-*/
-int sqlite3_expert_config(sqlite3expert *p, int op, ...);
-
-#define EXPERT_CONFIG_SAMPLE 1    /* int */
-
-/*
-** Specify zero or more SQL statements to be included in the analysis.
-**
-** Buffer zSql must contain zero or more complete SQL statements. This
-** function parses all statements contained in the buffer and adds them
-** to the internal list of statements to analyze. If successful, SQLITE_OK
-** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example
-** due to a error in the SQL - an SQLite error code is returned and (*pzErr)
-** may be set to point to an English language error message. In this case
-** the caller is responsible for eventually freeing the error message buffer
-** using sqlite3_free().
-**
-** If an error does occur while processing one of the statements in the
-** buffer passed as the second argument, none of the statements in the
-** buffer are added to the analysis.
-**
-** This function must be called before sqlite3_expert_analyze(). If a call
-** to this function is made on an sqlite3expert object that has already
-** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned
-** immediately and no statements are added to the analysis.
-*/
-int sqlite3_expert_sql(
-  sqlite3expert *p,               /* From a successful sqlite3_expert_new() */
-  const char *zSql,               /* SQL statement(s) to add */
-  char **pzErr                    /* OUT: Error message (if any) */
-);
-
-
-/*
-** This function is called after the sqlite3expert object has been configured
-** with all SQL statements using sqlite3_expert_sql() to actually perform
-** the analysis. Once this function has been called, it is not possible to
-** add further SQL statements to the analysis.
-**
-** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if
-** an error occurs, an SQLite error code is returned and (*pzErr) set to 
-** point to a buffer containing an English language error message. In this
-** case it is the responsibility of the caller to eventually free the buffer
-** using sqlite3_free().
-**
-** If an error does occur within this function, the sqlite3expert object
-** is no longer useful for any purpose. At that point it is no longer
-** possible to add further SQL statements to the object or to re-attempt
-** the analysis. The sqlite3expert object must still be freed using a call
-** sqlite3_expert_destroy().
-*/
-int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
-
-/*
-** Return the total number of statements loaded using sqlite3_expert_sql().
-** The total number of SQL statements may be different from the total number
-** to calls to sqlite3_expert_sql().
-*/
-int sqlite3_expert_count(sqlite3expert*);
-
-/*
-** Return a component of the report.
-**
-** This function is called after sqlite3_expert_analyze() to extract the
-** results of the analysis. Each call to this function returns either a
-** NULL pointer or a pointer to a buffer containing a nul-terminated string.
-** The value passed as the third argument must be one of the EXPERT_REPORT_*
-** #define constants defined below.
-**
-** For some EXPERT_REPORT_* parameters, the buffer returned contains 
-** information relating to a specific SQL statement. In these cases that
-** SQL statement is identified by the value passed as the second argument.
-** SQL statements are numbered from 0 in the order in which they are parsed.
-** If an out-of-range value (less than zero or equal to or greater than the
-** value returned by sqlite3_expert_count()) is passed as the second argument
-** along with such an EXPERT_REPORT_* parameter, NULL is always returned.
-**
-** EXPERT_REPORT_SQL:
-**   Return the text of SQL statement iStmt.
-**
-** EXPERT_REPORT_INDEXES:
-**   Return a buffer containing the CREATE INDEX statements for all recommended
-**   indexes for statement iStmt. If there are no new recommeded indexes, NULL 
-**   is returned.
-**
-** EXPERT_REPORT_PLAN:
-**   Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query
-**   iStmt after the proposed indexes have been added to the database schema.
-**
-** EXPERT_REPORT_CANDIDATES:
-**   Return a pointer to a buffer containing the CREATE INDEX statements 
-**   for all indexes that were tested (for all SQL statements). The iStmt
-**   parameter is ignored for EXPERT_REPORT_CANDIDATES calls.
-*/
-const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
-
-/*
-** Values for the third argument passed to sqlite3_expert_report().
-*/
-#define EXPERT_REPORT_SQL        1
-#define EXPERT_REPORT_INDEXES    2
-#define EXPERT_REPORT_PLAN       3
-#define EXPERT_REPORT_CANDIDATES 4
-
-/*
-** Free an (sqlite3expert*) handle and all associated resources. There 
-** should be one call to this function for each successful call to 
-** sqlite3-expert_new().
-*/
-void sqlite3_expert_destroy(sqlite3expert*);
-
-#endif  /* !defined(SQLITEEXPERT_H) */
@@ -1,220 +0,0 @@
-/*
-** 2017 April 07
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-*/
-
-#if defined(SQLITE_TEST)
-
-#include "sqlite3expert.h"
-#include <assert.h>
-#include <string.h>
-
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
-
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-
-/*
-** Extract an sqlite3* db handle from the object passed as the second
-** argument. If successful, set *pDb to point to the db handle and return
-** TCL_OK. Otherwise, return TCL_ERROR.
-*/
-static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
-  Tcl_CmdInfo info;
-  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
-    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
-    return TCL_ERROR;
-  }
-
-  *pDb = *(sqlite3 **)info.objClientData;
-  return TCL_OK;
-}
-
-
-/*
-** Tclcmd:  $expert sql SQL
-**          $expert analyze
-**          $expert count
-**          $expert report STMT EREPORT
-**          $expert destroy
-*/
-static int SQLITE_TCLAPI testExpertCmd(
-  void *clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  sqlite3expert *pExpert = (sqlite3expert*)clientData;
-  struct Subcmd {
-    const char *zSub;
-    int nArg;
-    const char *zMsg;
-  } aSub[] = {
-    { "sql",       1, "TABLE",        }, /* 0 */
-    { "analyze",   0, "",             }, /* 1 */
-    { "count",     0, "",             }, /* 2 */
-    { "report",    2, "STMT EREPORT", }, /* 3 */
-    { "destroy",   0, "",             }, /* 4 */
-    { 0 }
-  };
-  int iSub;
-  int rc = TCL_OK;
-  char *zErr = 0;
-
-  if( objc<2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
-    return TCL_ERROR;
-  }
-  rc = Tcl_GetIndexFromObjStruct(interp, 
-      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
-  );
-  if( rc!=TCL_OK ) return rc;
-  if( objc!=2+aSub[iSub].nArg ){
-    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
-    return TCL_ERROR;
-  }
-
-  switch( iSub ){
-    case 0: {      /* sql */
-      char *zArg = Tcl_GetString(objv[2]);
-      rc = sqlite3_expert_sql(pExpert, zArg, &zErr);
-      break;
-    }
-
-    case 1: {      /* analyze */
-      rc = sqlite3_expert_analyze(pExpert, &zErr);
-      break;
-    }
-
-    case 2: {      /* count */
-      int n = sqlite3_expert_count(pExpert);
-      Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
-      break;
-    }
-
-    case 3: {      /* report */
-      const char *aEnum[] = {
-        "sql", "indexes", "plan", "candidates", 0
-      };
-      int iEnum;
-      int iStmt;
-      const char *zReport;
-
-      if( Tcl_GetIntFromObj(interp, objv[2], &iStmt) 
-       || Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum)
-      ){
-        return TCL_ERROR;
-      }
-
-      assert( EXPERT_REPORT_SQL==1 );
-      assert( EXPERT_REPORT_INDEXES==2 );
-      assert( EXPERT_REPORT_PLAN==3 );
-      assert( EXPERT_REPORT_CANDIDATES==4 );
-      zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
-      Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
-      break;
-    }
-
-    default:       /* destroy */
-      assert( iSub==4 );     
-      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
-      break;
-  }
-
-  if( rc!=TCL_OK ){
-    if( zErr ){
-      Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
-    }else{
-      extern const char *sqlite3ErrName(int);
-      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
-    }
-  }
-  sqlite3_free(zErr);
-  return rc;
-}
-
-static void SQLITE_TCLAPI testExpertDel(void *clientData){
-  sqlite3expert *pExpert = (sqlite3expert*)clientData;
-  sqlite3_expert_destroy(pExpert);
-}
-
-/*
-** sqlite3_expert_new DB
-*/
-static int SQLITE_TCLAPI test_sqlite3_expert_new(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  static int iCmd = 0;
-  sqlite3 *db;
-  char *zCmd = 0;
-  char *zErr = 0;
-  sqlite3expert *pExpert;
-  int rc = TCL_OK;
-
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB");
-    return TCL_ERROR;
-  }
-  if( dbHandleFromObj(interp, objv[1], &db) ){
-    return TCL_ERROR;
-  }
-
-  zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd);
-  if( zCmd==0 ){
-    Tcl_AppendResult(interp, "out of memory", (char*)0);
-    return TCL_ERROR;
-  }
-
-  pExpert = sqlite3_expert_new(db, &zErr);
-  if( pExpert==0 ){
-    Tcl_AppendResult(interp, zErr, (char*)0);
-    rc = TCL_ERROR;
-  }else{
-    void *p = (void*)pExpert;
-    Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel);
-    Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1));
-  }
-
-  sqlite3_free(zCmd);
-  sqlite3_free(zErr);
-  return rc;
-}
-
-#endif  /* ifndef SQLITE_OMIT_VIRTUALTABLE */
-
-int TestExpert_Init(Tcl_Interp *interp){
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-  struct Cmd {
-    const char *zCmd;
-    Tcl_ObjCmdProc *xProc;
-  } aCmd[] = {
-    { "sqlite3_expert_new", test_sqlite3_expert_new },
-  };
-  int i;
-
-  for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
-    struct Cmd *p = &aCmd[i];
-    Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
-  }
-#endif
-  return TCL_OK;
-}
-
-#endif
@@ -52,10 +52,8 @@

  SECURITY: If the fts3 extension is used in an environment where potentially
    malicious users may execute arbitrary SQL (i.e. gears), they should be
-    prevented from invoking the fts3_tokenizer() function.  The
-    fts3_tokenizer() function is disabled by default. It is only enabled
-    by SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER. Do not enable it in
-    security sensitive environments.
+    prevented from invoking the fts3_tokenizer() function, possibly using the
+    authorisation callback.

  See "Sample code" below for an example of calling the fts3_tokenizer()
  function from C code.
@@ -95,8 +95,6 @@ SQLITE_EXTENSION_INIT3
 */
 #define FTS3_VARINT_MAX 10

-#define FTS3_BUFFER_PADDING 8
-
 /*
 ** FTS4 virtual tables may maintain multiple indexes - one index of all terms
 ** in the document set and zero or more prefix indexes. All indexes are stored
@@ -129,18 +127,6 @@ SQLITE_EXTENSION_INIT3
 #define POS_COLUMN  (1)     /* Column-list terminator */
 #define POS_END     (0)     /* Position-list terminator */ 

-/*
-** The assert_fts3_nc() macro is similar to the assert() macro, except that it
-** is used for assert() conditions that are true only if it can be 
-** guranteed that the database is not corrupt.
-*/
-#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
-extern int sqlite3_fts3_may_be_corrupt;
-# define assert_fts3_nc(x) assert(sqlite3_fts3_may_be_corrupt || (x))
-#else
-# define assert_fts3_nc(x) assert(x)
-#endif
-
 /*
 ** This section provides definitions to allow the
 ** FTS3 extension to be compiled outside of the 
@@ -196,9 +182,6 @@ typedef sqlite3_int64 i64;        /* 8-byte signed integer */
 # define TESTONLY(X)
 #endif

-#define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
-#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
-
 #endif /* SQLITE_AMALGAMATION */

 #ifdef SQLITE_DEBUG
@@ -242,13 +225,11 @@ struct Fts3Table {
  char *zLanguageid;              /* languageid=xxx option, or NULL */
  int nAutoincrmerge;             /* Value configured by 'automerge' */
  u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
-  int bLock;                      /* Used to prevent recursive content= tbls */

  /* Precompiled statements used by the implementation. Each of these 
  ** statements is run and reset within a single virtual table API call. 
  */
  sqlite3_stmt *aStmt[40];
-  sqlite3_stmt *pSeekStmt;        /* Cache for fts3CursorSeekStmt() */

  char *zReadExprlist;
  char *zWriteExprlist;
@@ -301,23 +282,13 @@ struct Fts3Table {
  int mxSavepoint;       /* Largest valid xSavepoint integer */
 #endif

-#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
+#ifdef SQLITE_TEST
  /* True to disable the incremental doclist optimization. This is controled
  ** by special insert command 'test-no-incr-doclist'.  */
  int bNoIncrDoclist;
-
-  /* Number of segments in a level */
-  int nMergeCount;
 #endif
 };

-/* Macro to find the number of segments to merge */
-#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
-# define MergeCount(P) ((P)->nMergeCount)
-#else
-# define MergeCount(P) FTS3_MERGE_COUNT
-#endif
-
 /*
 ** When the core wants to read from the virtual table, it creates a
 ** virtual table cursor (an instance of the following structure) using
@@ -328,7 +299,6 @@ struct Fts3Cursor {
  i16 eSearch;                    /* Search strategy (see below) */
  u8 isEof;                       /* True if at End Of Results */
  u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
-  u8 bSeekStmt;                   /* True if pStmt is a seek */
  sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
  Fts3Expr *pExpr;                /* Parsed MATCH query string */
  int iLangid;                    /* Language being queried for */
@@ -581,8 +551,6 @@ int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
 void sqlite3Fts3ErrMsg(char**,const char*,...);
 int sqlite3Fts3PutVarint(char *, sqlite3_int64);
 int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
-int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *);
-int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*);
 int sqlite3Fts3GetVarint32(const char *, int *);
 int sqlite3Fts3VarintLen(sqlite3_uint64);
 void sqlite3Fts3Dequote(char *);
@@ -591,7 +559,6 @@ int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
 int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
 void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
 int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc);
-int sqlite3Fts3ReadInt(const char *z, int *pnOut);

 /* fts3_tokenizer.c */
 const char *sqlite3Fts3NextToken(const char *, int *);
@@ -615,7 +582,7 @@ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
 );
 void sqlite3Fts3ExprFree(Fts3Expr *);
 #ifdef SQLITE_TEST
-int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
 int sqlite3Fts3InitTerm(sqlite3 *db);
 #endif

@@ -66,7 +66,7 @@ static int fts3auxConnectMethod(
  char const *zFts3;              /* Name of fts3 table */
  int nDb;                        /* Result of strlen(zDb) */
  int nFts3;                      /* Result of strlen(zFts3) */
-  sqlite3_int64 nByte;            /* Bytes of space to allocate here */
+  int nByte;                      /* Bytes of space to allocate here */
  int rc;                         /* value returned by declare_vtab() */
  Fts3auxTable *p;                /* Virtual table object to return */

@@ -98,7 +98,7 @@ static int fts3auxConnectMethod(
  if( rc!=SQLITE_OK ) return rc;

  nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
-  p = (Fts3auxTable *)sqlite3_malloc64(nByte);
+  p = (Fts3auxTable *)sqlite3_malloc(nByte);
  if( !p ) return SQLITE_NOMEM;
  memset(p, 0, nByte);

@@ -248,7 +248,7 @@ static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){
 static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){
  if( nSize>pCsr->nStat ){
    struct Fts3auxColstats *aNew;
-    aNew = (struct Fts3auxColstats *)sqlite3_realloc64(pCsr->aStat, 
+    aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, 
        sizeof(struct Fts3auxColstats) * nSize
    );
    if( aNew==0 ) return SQLITE_NOMEM;
@@ -416,15 +416,15 @@ static int fts3auxFilterMethod(
    assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) );
    if( zStr ){
      pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr);
+      pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]);
      if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM;
-      pCsr->filter.nTerm = (int)strlen(pCsr->filter.zTerm);
    }
  }

  if( iLe>=0 ){
    pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe]));
+    pCsr->nStop = sqlite3_value_bytes(apVal[iLe]);
    if( pCsr->zStop==0 ) return SQLITE_NOMEM;
-    pCsr->nStop = (int)strlen(pCsr->zStop);
  }
  
  if( iLangid>=0 ){
@@ -539,8 +539,7 @@ int sqlite3Fts3InitAux(sqlite3 *db){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0                            /* xRollbackTo   */
  };
  int rc;                         /* Return code */

@@ -122,8 +122,8 @@ static int fts3isspace(char c){
 ** zero the memory before returning a pointer to it. If unsuccessful, 
 ** return NULL.
 */
-static void *fts3MallocZero(sqlite3_int64 nByte){
-  void *pRet = sqlite3_malloc64(nByte);
+static void *fts3MallocZero(int nByte){
+  void *pRet = sqlite3_malloc(nByte);
  if( pRet ) memset(pRet, 0, nByte);
  return pRet;
 }
@@ -198,7 +198,7 @@ static int getNextToken(
  if( rc==SQLITE_OK ){
    const char *zToken;
    int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
-    sqlite3_int64 nByte;                    /* total space to allocate */
+    int nByte;                               /* total space to allocate */

    rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
    if( rc==SQLITE_OK ){
@@ -252,8 +252,8 @@ static int getNextToken(
 ** Enlarge a memory allocation.  If an out-of-memory allocation occurs,
 ** then free the old allocation.
 */
-static void *fts3ReallocOrFree(void *pOrig, sqlite3_int64 nNew){
-  void *pRet = sqlite3_realloc64(pOrig, nNew);
+static void *fts3ReallocOrFree(void *pOrig, int nNew){
+  void *pRet = sqlite3_realloc(pOrig, nNew);
  if( !pRet ){
    sqlite3_free(pOrig);
  }
@@ -446,7 +446,10 @@ static int getNextNode(
      if( pKey->eType==FTSQUERY_NEAR ){
        assert( nKey==4 );
        if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){
-          nKey += 1+sqlite3Fts3ReadInt(&zInput[nKey+1], &nNear);
+          nNear = 0;
+          for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){
+            nNear = nNear * 10 + (zInput[nKey] - '0');
+          }
        }
      }

@@ -494,6 +497,7 @@ static int getNextNode(
      int nConsumed = 0;
      pParse->nNest++;
      rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed);
+      if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; }
      *pnConsumed = (int)(zInput - z) + 1 + nConsumed;
      return rc;
    }else if( *zInput==')' ){
@@ -792,7 +796,7 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){
  if( rc==SQLITE_OK ){
    if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
      Fts3Expr **apLeaf;
-      apLeaf = (Fts3Expr **)sqlite3_malloc64(sizeof(Fts3Expr *) * nMaxDepth);
+      apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
      if( 0==apLeaf ){
        rc = SQLITE_NOMEM;
      }else{
@@ -1104,6 +1108,34 @@ void sqlite3Fts3ExprFree(Fts3Expr *pDel){

 #include <stdio.h>

+/*
+** Function to query the hash-table of tokenizers (see README.tokenizers).
+*/
+static int queryTestTokenizer(
+  sqlite3 *db, 
+  const char *zName,  
+  const sqlite3_tokenizer_module **pp
+){
+  int rc;
+  sqlite3_stmt *pStmt;
+  const char zSql[] = "SELECT fts3_tokenizer(?)";
+
+  *pp = 0;
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+  if( SQLITE_ROW==sqlite3_step(pStmt) ){
+    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+    }
+  }
+
+  return sqlite3_finalize(pStmt);
+}
+
 /*
 ** Return a pointer to a buffer containing a text representation of the
 ** expression passed as the first argument. The buffer is obtained from
@@ -1171,12 +1203,12 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){
 **
 **   SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
 */
-static void fts3ExprTestCommon(
-  int bRebalance,
+static void fts3ExprTest(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
 ){
+  sqlite3_tokenizer_module const *pModule = 0;
  sqlite3_tokenizer *pTokenizer = 0;
  int rc;
  char **azCol = 0;
@@ -1186,9 +1218,7 @@ static void fts3ExprTestCommon(
  int ii;
  Fts3Expr *pExpr;
  char *zBuf = 0;
-  Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context);
-  const char *zTokenizer = 0;
-  char *zErr = 0;
+  sqlite3 *db = sqlite3_context_db_handle(context);

  if( argc<3 ){
    sqlite3_result_error(context, 
@@ -1197,22 +1227,28 @@ static void fts3ExprTestCommon(
    return;
  }

-  zTokenizer = (const char*)sqlite3_value_text(argv[0]);
-  rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr);
-  if( rc!=SQLITE_OK ){
-    if( rc==SQLITE_NOMEM ){
-      sqlite3_result_error_nomem(context);
-    }else{
-      sqlite3_result_error(context, zErr, -1);
-    }
-    sqlite3_free(zErr);
-    return;
+  rc = queryTestTokenizer(db,
+                          (const char *)sqlite3_value_text(argv[0]), &pModule);
+  if( rc==SQLITE_NOMEM ){
+    sqlite3_result_error_nomem(context);
+    goto exprtest_out;
+  }else if( !pModule ){
+    sqlite3_result_error(context, "No such tokenizer module", -1);
+    goto exprtest_out;
  }

+  rc = pModule->xCreate(0, 0, &pTokenizer);
+  assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
+  if( rc==SQLITE_NOMEM ){
+    sqlite3_result_error_nomem(context);
+    goto exprtest_out;
+  }
+  pTokenizer->pModule = pModule;
+
  zExpr = (const char *)sqlite3_value_text(argv[1]);
  nExpr = sqlite3_value_bytes(argv[1]);
  nCol = argc-2;
-  azCol = (char **)sqlite3_malloc64(nCol*sizeof(char *));
+  azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
  if( !azCol ){
    sqlite3_result_error_nomem(context);
    goto exprtest_out;
@@ -1221,7 +1257,7 @@ static void fts3ExprTestCommon(
    azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
  }

-  if( bRebalance ){
+  if( sqlite3_user_data(context) ){
    char *zDummy = 0;
    rc = sqlite3Fts3ExprParse(
        pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy
@@ -1247,38 +1283,23 @@ static void fts3ExprTestCommon(
  sqlite3Fts3ExprFree(pExpr);

 exprtest_out:
-  if( pTokenizer ){
-    rc = pTokenizer->pModule->xDestroy(pTokenizer);
+  if( pModule && pTokenizer ){
+    rc = pModule->xDestroy(pTokenizer);
  }
  sqlite3_free(azCol);
 }

-static void fts3ExprTest(
-  sqlite3_context *context,
-  int argc,
-  sqlite3_value **argv
-){
-  fts3ExprTestCommon(0, context, argc, argv);
-}
-static void fts3ExprTestRebalance(
-  sqlite3_context *context,
-  int argc,
-  sqlite3_value **argv
-){
-  fts3ExprTestCommon(1, context, argc, argv);
-}
-
 /*
 ** Register the query expression parser test function fts3_exprtest() 
 ** with database connection db. 
 */
-int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){
+int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
  int rc = sqlite3_create_function(
-      db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0
+      db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
  );
  if( rc==SQLITE_OK ){
    rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", 
-        -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0
+        -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
    );
  }
  return rc;
@@ -35,8 +35,8 @@
 /*
 ** Malloc and Free functions
 */
-static void *fts3HashMalloc(sqlite3_int64 n){
-  void *p = sqlite3_malloc64(n);
+static void *fts3HashMalloc(int n){
+  void *p = sqlite3_malloc(n);
  if( p ){
    memset(p, 0, n);
  }
@@ -60,7 +60,7 @@ static int icuCreate(
  if( argc>0 ){
    n = strlen(argv[0])+1;
  }
-  p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n);
+  p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
  if( !p ){
    return SQLITE_NOMEM;
  }
@@ -117,7 +117,7 @@ static int icuOpen(
    nInput = strlen(zInput);
  }
  nChar = nInput+1;
-  pCsr = (IcuCursor *)sqlite3_malloc64(
+  pCsr = (IcuCursor *)sqlite3_malloc(
      sizeof(IcuCursor) +                /* IcuCursor */
      ((nChar+3)&~3) * sizeof(UChar) +   /* IcuCursor.aChar[] */
      (nChar+1) * sizeof(int)            /* IcuCursor.aOffset[] */
@@ -128,19 +128,17 @@ struct StrBuffer {
 /*
 ** Allocate a two-slot MatchinfoBuffer object.
 */
-static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){
+static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){
  MatchinfoBuffer *pRet;
-  sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1)
-                           + sizeof(MatchinfoBuffer);
-  sqlite3_int64 nStr = strlen(zMatchinfo);
+  int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer);
+  int nStr = (int)strlen(zMatchinfo);

-  pRet = sqlite3_malloc64(nByte + nStr+1);
+  pRet = sqlite3_malloc(nByte + nStr+1);
  if( pRet ){
    memset(pRet, 0, nByte);
    pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet;
-    pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0]
-                                      + sizeof(u32)*((int)nElem+1);
-    pRet->nElem = (int)nElem;
+    pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1);
+    pRet->nElem = nElem;
    pRet->zMatchinfo = ((char*)pRet) + nByte;
    memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1);
    pRet->aRef[0] = 1;
@@ -180,7 +178,7 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){
    aOut = &p->aMatchinfo[p->nElem+2];
    xRet = fts3MIBufferFree;
  }else{
-    aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32));
+    aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32));
    if( aOut ){
      xRet = sqlite3_free;
      if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32));
@@ -431,12 +429,11 @@ static void fts3SnippetDetails(
      char *pCsr = pPhrase->pTail;
      int iCsr = pPhrase->iTail;

-      while( iCsr<(iStart+pIter->nSnippet) && iCsr>=iStart ){
+      while( iCsr<(iStart+pIter->nSnippet) ){
        int j;
-        u64 mPhrase = (u64)1 << (i%64);
+        u64 mPhrase = (u64)1 << i;
        u64 mPos = (u64)1 << (iCsr - iStart);
-        assert( iCsr>=iStart && (iCsr - iStart)<=64 );
-        assert( i>=0 );
+        assert( iCsr>=iStart );
        if( (mCover|mCovered)&mPhrase ){
          iScore++;
        }else{
@@ -478,14 +475,11 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
    int iFirst = 0;
    pPhrase->pList = pCsr;
    fts3GetDeltaPosition(&pCsr, &iFirst);
-    if( iFirst<0 ){
-      rc = FTS_CORRUPT_VTAB;
-    }else{
-      pPhrase->pHead = pCsr;
-      pPhrase->pTail = pCsr;
-      pPhrase->iHead = iFirst;
-      pPhrase->iTail = iFirst;
-    }
+    assert( iFirst>=0 );
+    pPhrase->pHead = pCsr;
+    pPhrase->pTail = pCsr;
+    pPhrase->iHead = iFirst;
+    pPhrase->iTail = iFirst;
  }else{
    assert( rc!=SQLITE_OK || (
       pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 
@@ -522,7 +516,7 @@ static int fts3BestSnippet(
  int rc;                         /* Return Code */
  int nList;                      /* Number of phrases in expression */
  SnippetIter sIter;              /* Iterates through snippet candidates */
-  sqlite3_int64 nByte;            /* Number of bytes of space to allocate */
+  int nByte;                      /* Number of bytes of space to allocate */
  int iBestScore = -1;            /* Best snippet score found so far */
  int i;                          /* Loop counter */

@@ -540,7 +534,7 @@ static int fts3BestSnippet(
  ** the required space using malloc().
  */
  nByte = sizeof(SnippetPhrase) * nList;
-  sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc64(nByte);
+  sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte);
  if( !sIter.aPhrase ){
    return SQLITE_NOMEM;
  }
@@ -560,7 +554,7 @@ static int fts3BestSnippet(
    /* Set the *pmSeen output variable. */
    for(i=0; i<nList; i++){
      if( sIter.aPhrase[i].pHead ){
-        *pmSeen |= (u64)1 << (i%64);
+        *pmSeen |= (u64)1 << i;
      }
    }

@@ -610,8 +604,8 @@ static int fts3StringAppend(
  ** appended data.
  */
  if( pStr->n+nAppend+1>=pStr->nAlloc ){
-    sqlite3_int64 nAlloc = pStr->nAlloc+(sqlite3_int64)nAppend+100;
-    char *zNew = sqlite3_realloc64(pStr->z, nAlloc);
+    int nAlloc = pStr->nAlloc+nAppend+100;
+    char *zNew = sqlite3_realloc(pStr->z, nAlloc);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
@@ -666,7 +660,6 @@ static int fts3SnippetShift(

    for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++);
    for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++);
-    assert( (nSnippet-1-nRight)<=63 && (nSnippet-1-nRight)>=0 );
    nDesired = (nLeft-nRight)/2;

    /* Ideally, the start of the snippet should be pushed forward in the
@@ -859,7 +852,7 @@ static int fts3ColumnlistCount(char **ppCollist){
 /*
 ** This function gathers 'y' or 'b' data for a single phrase.
 */
-static int fts3ExprLHits(
+static void fts3ExprLHits(
  Fts3Expr *pExpr,                /* Phrase expression node */
  MatchInfo *p                    /* Matchinfo context */
 ){
@@ -889,29 +882,25 @@ static int fts3ExprLHits(
    if( *pIter!=0x01 ) break;
    pIter++;
    pIter += fts3GetVarint32(pIter, &iCol);
-    if( iCol>=p->nCol ) return FTS_CORRUPT_VTAB;
  }
-  return SQLITE_OK;
 }

 /*
 ** Gather the results for matchinfo directives 'y' and 'b'.
 */
-static int fts3ExprLHitGather(
+static void fts3ExprLHitGather(
  Fts3Expr *pExpr,
  MatchInfo *p
 ){
-  int rc = SQLITE_OK;
  assert( (pExpr->pLeft==0)==(pExpr->pRight==0) );
  if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
    if( pExpr->pLeft ){
-      rc = fts3ExprLHitGather(pExpr->pLeft, p);
-      if( rc==SQLITE_OK ) rc = fts3ExprLHitGather(pExpr->pRight, p);
+      fts3ExprLHitGather(pExpr->pLeft, p);
+      fts3ExprLHitGather(pExpr->pRight, p);
    }else{
-      rc = fts3ExprLHits(pExpr, p);
+      fts3ExprLHits(pExpr, p);
    }
  }
-  return rc;
 }

 /*
@@ -1001,8 +990,8 @@ static int fts3MatchinfoCheck(
  return SQLITE_ERROR;
 }

-static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
-  size_t nVal;                      /* Number of integers output by cArg */
+static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
+  int nVal;                       /* Number of integers output by cArg */

  switch( cArg ){
    case FTS3_MATCHINFO_NDOC:
@@ -1038,15 +1027,11 @@ static int fts3MatchinfoSelectDoctotal(
  Fts3Table *pTab,
  sqlite3_stmt **ppStmt,
  sqlite3_int64 *pnDoc,
-  const char **paLen,
-  const char **ppEnd
+  const char **paLen
 ){
  sqlite3_stmt *pStmt;
  const char *a;
-  const char *pEnd;
  sqlite3_int64 nDoc;
-  int n;
-

  if( !*ppStmt ){
    int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
@@ -1055,20 +1040,12 @@ static int fts3MatchinfoSelectDoctotal(
  pStmt = *ppStmt;
  assert( sqlite3_data_count(pStmt)==1 );

-  n = sqlite3_column_bytes(pStmt, 0);
  a = sqlite3_column_blob(pStmt, 0);
-  if( a==0 ){
-    return FTS_CORRUPT_VTAB;
-  }
-  pEnd = a + n;
-  a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc);
-  if( nDoc<=0 || a>pEnd ){
-    return FTS_CORRUPT_VTAB;
-  }
-  *pnDoc = nDoc;
+  a += sqlite3Fts3GetVarint(a, &nDoc);
+  if( nDoc==0 ) return FTS_CORRUPT_VTAB;
+  *pnDoc = (u32)nDoc;

  if( paLen ) *paLen = a;
-  if( ppEnd ) *ppEnd = pEnd;
  return SQLITE_OK;
 }

@@ -1140,12 +1117,11 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
  int i;
  int iCol;
  int nToken = 0;
-  int rc = SQLITE_OK;

  /* Allocate and populate the array of LcsIterator objects. The array
  ** contains one element for each matchable phrase in the query.
  **/
-  aIter = sqlite3_malloc64(sizeof(LcsIterator) * pCsr->nPhrase);
+  aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
  if( !aIter ) return SQLITE_NOMEM;
  memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
  (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
@@ -1161,16 +1137,13 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
    int nLive = 0;                /* Number of iterators in aIter not at EOF */

    for(i=0; i<pInfo->nPhrase; i++){
+      int rc;
      LcsIterator *pIt = &aIter[i];
      rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead);
-      if( rc!=SQLITE_OK ) goto matchinfo_lcs_out;
+      if( rc!=SQLITE_OK ) return rc;
      if( pIt->pRead ){
        pIt->iPos = pIt->iPosOffset;
-        fts3LcsIteratorAdvance(pIt);
-        if( pIt->pRead==0 ){
-          rc = FTS_CORRUPT_VTAB;
-          goto matchinfo_lcs_out;
-        }
+        fts3LcsIteratorAdvance(&aIter[i]);
        nLive++;
      }
    }
@@ -1202,9 +1175,8 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
    pInfo->aMatchinfo[iCol] = nLcs;
  }

- matchinfo_lcs_out:
  sqlite3_free(aIter);
-  return rc;
+  return SQLITE_OK;
 }

 /*
@@ -1249,7 +1221,7 @@ static int fts3MatchinfoValues(
      case FTS3_MATCHINFO_NDOC:
        if( bGlobal ){
          sqlite3_int64 nDoc = 0;
-          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0, 0);
+          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0);
          pInfo->aMatchinfo[0] = (u32)nDoc;
        }
        break;
@@ -1258,19 +1230,14 @@ static int fts3MatchinfoValues(
        if( bGlobal ){
          sqlite3_int64 nDoc;     /* Number of rows in table */
          const char *a;          /* Aggregate column length array */
-          const char *pEnd;       /* First byte past end of length array */

-          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a, &pEnd);
+          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a);
          if( rc==SQLITE_OK ){
            int iCol;
            for(iCol=0; iCol<pInfo->nCol; iCol++){
              u32 iVal;
              sqlite3_int64 nToken;
              a += sqlite3Fts3GetVarint(a, &nToken);
-              if( a>pEnd ){
-                rc = SQLITE_CORRUPT_VTAB;
-                break;
-              }
              iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc);
              pInfo->aMatchinfo[iCol] = iVal;
            }
@@ -1284,14 +1251,9 @@ static int fts3MatchinfoValues(
        if( rc==SQLITE_OK ){
          int iCol;
          const char *a = sqlite3_column_blob(pSelectDocsize, 0);
-          const char *pEnd = a + sqlite3_column_bytes(pSelectDocsize, 0);
          for(iCol=0; iCol<pInfo->nCol; iCol++){
            sqlite3_int64 nToken;
-            a += sqlite3Fts3GetVarintBounded(a, pEnd, &nToken);
-            if( a>pEnd ){
-              rc = SQLITE_CORRUPT_VTAB;
-              break;
-            }
+            a += sqlite3Fts3GetVarint(a, &nToken);
            pInfo->aMatchinfo[iCol] = (u32)nToken;
          }
        }
@@ -1308,9 +1270,9 @@ static int fts3MatchinfoValues(

      case FTS3_MATCHINFO_LHITS_BM:
      case FTS3_MATCHINFO_LHITS: {
-        size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
+        int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
        memset(pInfo->aMatchinfo, 0, nZero);
-        rc = fts3ExprLHitGather(pCsr->pExpr, pInfo);
+        fts3ExprLHitGather(pCsr->pExpr, pInfo);
        break;
      }

@@ -1322,7 +1284,7 @@ static int fts3MatchinfoValues(
        if( rc!=SQLITE_OK ) break;
        if( bGlobal ){
          if( pCsr->pDeferred ){
-            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0);
+            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
            if( rc!=SQLITE_OK ) break;
          }
          rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
@@ -1377,7 +1339,7 @@ static void fts3GetMatchinfo(
  ** initialize those elements that are constant for every row.
  */
  if( pCsr->pMIBuffer==0 ){
-    size_t nMatchinfo = 0;        /* Number of u32 elements in match-info */
+    int nMatchinfo = 0;           /* Number of u32 elements in match-info */
    int i;                        /* Used to iterate through zArg */

    /* Determine the number of phrases in the query */
@@ -1462,10 +1424,6 @@ void sqlite3Fts3Snippet(
    return;
  }

-  /* Limit the snippet length to 64 tokens. */
-  if( nToken<-64 ) nToken = -64;
-  if( nToken>+64 ) nToken = +64;
-
  for(nSnippet=1; 1; nSnippet++){

    int iSnip;                    /* Loop counter 0..nSnippet-1 */
@@ -1567,7 +1525,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
  nTerm = pExpr->pPhrase->nToken;
  if( pList ){
    fts3GetDeltaPosition(&pList, &iPos);
-    assert_fts3_nc( iPos>=0 );
+    assert( iPos>=0 );
  }

  for(iTerm=0; iTerm<nTerm; iTerm++){
@@ -1608,7 +1566,7 @@ void sqlite3Fts3Offsets(
  if( rc!=SQLITE_OK ) goto offsets_out;

  /* Allocate the array of TermOffset iterators. */
-  sCtx.aTerm = (TermOffset *)sqlite3_malloc64(sizeof(TermOffset)*nToken);
+  sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken);
  if( 0==sCtx.aTerm ){
    rc = SQLITE_NOMEM;
    goto offsets_out;
@@ -1677,7 +1635,7 @@ void sqlite3Fts3Offsets(
        /* All offsets for this column have been gathered. */
        rc = SQLITE_DONE;
      }else{
-        assert_fts3_nc( iCurrent<=iMinPos );
+        assert( iCurrent<=iMinPos );
        if( 0==(0xFE&*pTerm->pList) ){
          pTerm->pList = 0;
        }else{
@@ -68,9 +68,9 @@ static int fts3termConnectMethod(
  char const *zFts3;              /* Name of fts3 table */
  int nDb;                        /* Result of strlen(zDb) */
  int nFts3;                      /* Result of strlen(zFts3) */
-  sqlite3_int64 nByte;            /* Bytes of space to allocate here */
+  int nByte;                      /* Bytes of space to allocate here */
  int rc;                         /* value returned by declare_vtab() */
-  Fts3termTable *p;               /* Virtual table object to return */
+  Fts3termTable *p;                /* Virtual table object to return */
  int iIndex = 0;

  UNUSED_PARAMETER(pCtx);
@@ -96,9 +96,9 @@ static int fts3termConnectMethod(
  if( rc!=SQLITE_OK ) return rc;

  nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
-  p = (Fts3termTable *)sqlite3_malloc64(nByte);
+  p = (Fts3termTable *)sqlite3_malloc(nByte);
  if( !p ) return SQLITE_NOMEM;
-  memset(p, 0, (size_t)nByte);
+  memset(p, 0, nByte);

  p->pFts3Tab = (Fts3Table *)&p[1];
  p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
@@ -361,8 +361,7 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0                            /* xRollbackTo   */
  };
  int rc;                         /* Return code */

@@ -448,14 +448,14 @@ static int testTokenizerNext(
  }else{
    /* Advance to the end of the token */
    const char *pToken = p;
-    sqlite3_int64 nToken;
+    int nToken;
    while( p<pEnd && testIsTokenChar(*p) ) p++;
-    nToken = (sqlite3_int64)(p-pToken);
+    nToken = (int)(p-pToken);

    /* Copy the token into the buffer */
    if( nToken>pCsr->nBuffer ){
      sqlite3_free(pCsr->aBuffer);
-      pCsr->aBuffer = sqlite3_malloc64(nToken);
+      pCsr->aBuffer = sqlite3_malloc(nToken);
    }
    if( pCsr->aBuffer==0 ){
      rc = SQLITE_NOMEM;
@@ -471,7 +471,7 @@ static int testTokenizerNext(
      pCsr->iInput = (int)(p - pCsr->aInput);

      *ppToken = pCsr->aBuffer;
-      *pnBytes = (int)nToken;
+      *pnBytes = nToken;
      *piStartOffset = (int)(pToken - pCsr->aInput);
      *piEndOffset = (int)(p - pCsr->aInput);
      *piPosition = pCsr->iToken;
@@ -574,33 +574,6 @@ static int SQLITE_TCLAPI fts3_test_varint_cmd(
 ** End of tokenizer code.
 **************************************************************************/ 

-/*
-**      sqlite3_fts3_may_be_corrupt BOOLEAN
-**
-** Set or clear the global "may-be-corrupt" flag. Return the old value.
-*/
-static int SQLITE_TCLAPI fts3_may_be_corrupt(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  int bOld = sqlite3_fts3_may_be_corrupt;
-
-  if( objc!=2 && objc!=1 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?");
-    return TCL_ERROR;
-  }
-  if( objc==2 ){
-    int bNew;
-    if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR;
-    sqlite3_fts3_may_be_corrupt = bNew;
-  }
-
-  Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld));
-  return TCL_OK;
-}
-
 int Sqlitetestfts3_Init(Tcl_Interp *interp){
  Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
  Tcl_CreateObjCommand(interp, 
@@ -609,12 +582,10 @@ int Sqlitetestfts3_Init(Tcl_Interp *interp){
  Tcl_CreateObjCommand(
      interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0
  );
+
  Tcl_CreateObjCommand(
      interp, "fts3_test_varint", fts3_test_varint_cmd, 0, 0
  );
-  Tcl_CreateObjCommand(
-      interp, "sqlite3_fts3_may_be_corrupt", fts3_may_be_corrupt, 0, 0
-  );
  return TCL_OK;
 }
 #endif                  /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */
@@ -122,7 +122,7 @@ static int fts3tokDequoteArray(
      nByte += (int)(strlen(argv[i]) + 1);
    }

-    *pazDequote = azDequote = sqlite3_malloc64(sizeof(char *)*argc + nByte);
+    *pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
    if( azDequote==0 ){
      rc = SQLITE_NOMEM;
    }else{
@@ -346,11 +346,11 @@ static int fts3tokFilterMethod(
  if( idxNum==1 ){
    const char *zByte = (const char *)sqlite3_value_text(apVal[0]);
    int nByte = sqlite3_value_bytes(apVal[0]);
-    pCsr->zInput = sqlite3_malloc64(nByte+1);
+    pCsr->zInput = sqlite3_malloc(nByte+1);
    if( pCsr->zInput==0 ){
      rc = SQLITE_NOMEM;
    }else{
-      if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte);
+      memcpy(pCsr->zInput, zByte, nByte);
      pCsr->zInput[nByte] = 0;
      rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr);
      if( rc==SQLITE_OK ){
@@ -443,8 +443,7 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0                            /* xRollbackTo   */
  };
  int rc;                         /* Return code */

@@ -79,7 +79,7 @@ static void fts3TokenizerFunc(
  nName = sqlite3_value_bytes(argv[0])+1;

  if( argc==2 ){
-    if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){
+    if( fts3TokenizerEnabled(context) ){
      void *pOld;
      int n = sqlite3_value_bytes(argv[1]);
      if( zName==0 || n!=sizeof(pPtr) ){
@@ -106,9 +106,7 @@ static void fts3TokenizerFunc(
      return;
    }
  }
-  if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){
-    sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
-  }
+  sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
 }

 int sqlite3Fts3IsIdChar(char c){
@@ -196,8 +194,8 @@ int sqlite3Fts3InitTokenizer(
    int iArg = 0;
    z = &z[n+1];
    while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
-      sqlite3_int64 nNew = sizeof(char *)*(iArg+1);
-      char const **aNew = (const char **)sqlite3_realloc64((void *)aArg, nNew);
+      int nNew = sizeof(char *)*(iArg+1);
+      char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew);
      if( !aNew ){
        sqlite3_free(zCopy);
        sqlite3_free((void *)aArg);
@@ -390,9 +388,7 @@ int queryTokenizer(

  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
  if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB
-     && sqlite3_column_bytes(pStmt, 0)==sizeof(*pp)
-    ){
+    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
    }
  }
@@ -481,7 +477,7 @@ int sqlite3Fts3InitHashTable(
 ){
  int rc = SQLITE_OK;
  void *p = (void *)pHash;
-  const int any = SQLITE_UTF8|SQLITE_DIRECTONLY;
+  const int any = SQLITE_ANY;

 #ifdef SQLITE_TEST
  char *zTest = 0;
@@ -82,7 +82,7 @@ typedef struct unicode_cursor unicode_cursor;

 struct unicode_tokenizer {
  sqlite3_tokenizer base;
-  int eRemoveDiacritic;
+  int bRemoveDiacritic;
  int nException;
  int *aiException;
 };
@@ -136,16 +136,16 @@ static int unicodeAddExceptions(
 ){
  const unsigned char *z = (const unsigned char *)zIn;
  const unsigned char *zTerm = &z[nIn];
-  unsigned int iCode;
+  int iCode;
  int nEntry = 0;

  assert( bAlnum==0 || bAlnum==1 );

  while( z<zTerm ){
    READ_UTF8(z, zTerm, iCode);
-    assert( (sqlite3FtsUnicodeIsalnum((int)iCode) & 0xFFFFFFFE)==0 );
-    if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 
-     && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0 
+    assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
+    if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 
+     && sqlite3FtsUnicodeIsdiacritic(iCode)==0 
    ){
      nEntry++;
    }
@@ -155,20 +155,20 @@ static int unicodeAddExceptions(
    int *aNew;                    /* New aiException[] array */
    int nNew;                     /* Number of valid entries in array aNew[] */

-    aNew = sqlite3_realloc64(p->aiException,(p->nException+nEntry)*sizeof(int));
+    aNew = sqlite3_realloc(p->aiException, (p->nException+nEntry)*sizeof(int));
    if( aNew==0 ) return SQLITE_NOMEM;
    nNew = p->nException;

    z = (const unsigned char *)zIn;
    while( z<zTerm ){
      READ_UTF8(z, zTerm, iCode);
-      if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 
-       && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
+      if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 
+       && sqlite3FtsUnicodeIsdiacritic(iCode)==0
      ){
        int i, j;
-        for(i=0; i<nNew && aNew[i]<(int)iCode; i++);
+        for(i=0; i<nNew && aNew[i]<iCode; i++);
        for(j=nNew; j>i; j--) aNew[j] = aNew[j-1];
-        aNew[i] = (int)iCode;
+        aNew[i] = iCode;
        nNew++;
      }
    }
@@ -227,20 +227,17 @@ static int unicodeCreate(
  pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
  if( pNew==NULL ) return SQLITE_NOMEM;
  memset(pNew, 0, sizeof(unicode_tokenizer));
-  pNew->eRemoveDiacritic = 1;
+  pNew->bRemoveDiacritic = 1;

  for(i=0; rc==SQLITE_OK && i<nArg; i++){
    const char *z = azArg[i];
    int n = (int)strlen(z);

    if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
-      pNew->eRemoveDiacritic = 1;
+      pNew->bRemoveDiacritic = 1;
    }
    else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
-      pNew->eRemoveDiacritic = 0;
-    }
-    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
-      pNew->eRemoveDiacritic = 2;
+      pNew->bRemoveDiacritic = 0;
    }
    else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
      rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
@@ -321,7 +318,7 @@ static int unicodeNext(
 ){
  unicode_cursor *pCsr = (unicode_cursor *)pC;
  unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer);
-  unsigned int iCode = 0;
+  int iCode = 0;
  char *zOut;
  const unsigned char *z = &pCsr->aInput[pCsr->iOff];
  const unsigned char *zStart = z;
@@ -333,7 +330,7 @@ static int unicodeNext(
  ** the input.  */
  while( z<zTerm ){
    READ_UTF8(z, zTerm, iCode);
-    if( unicodeIsAlnum(p, (int)iCode) ) break;
+    if( unicodeIsAlnum(p, iCode) ) break;
    zStart = z;
  }
  if( zStart>=zTerm ) return SQLITE_DONE;
@@ -344,7 +341,7 @@ static int unicodeNext(

    /* Grow the output buffer if required. */
    if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){
-      char *zNew = sqlite3_realloc64(pCsr->zToken, pCsr->nAlloc+64);
+      char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64);
      if( !zNew ) return SQLITE_NOMEM;
      zOut = &zNew[zOut - pCsr->zToken];
      pCsr->zToken = zNew;
@@ -353,7 +350,7 @@ static int unicodeNext(

    /* Write the folded case of the last character read to the output */
    zEnd = z;
-    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
+    iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic);
    if( iOut ){
      WRITE_UTF8(zOut, iOut);
    }
@@ -361,8 +358,8 @@ static int unicodeNext(
    /* If the cursor is not at EOF, read the next character */
    if( z>=zTerm ) break;
    READ_UTF8(z, zTerm, iCode);
-  }while( unicodeIsAlnum(p, (int)iCode) 
-       || sqlite3FtsUnicodeIsdiacritic((int)iCode)
+  }while( unicodeIsAlnum(p, iCode) 
+       || sqlite3FtsUnicodeIsdiacritic(iCode)
  );

  /* Set the output variables and return. */
@@ -1,5 +1,5 @@
 /*
-** 2012-05-25
+** 2012 May 25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -127,9 +127,9 @@ int sqlite3FtsUnicodeIsalnum(int c){
    0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
  };

-  if( (unsigned int)c<128 ){
-    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
-  }else if( (unsigned int)c<(1<<22) ){
+  if( c<128 ){
+    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
+  }else if( c<(1<<22) ){
    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
    int iRes = 0;
    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
@@ -159,48 +159,32 @@ int sqlite3FtsUnicodeIsalnum(int c){
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int remove_diacritic(int c, int bComplex){
+static int remove_diacritic(int c){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
-     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
-     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
-     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
-     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
-    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
-    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
-    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
-    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
-    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
-    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
-    63182, 63242, 63274, 63310, 63368, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
+     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
+     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
+     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
+    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
+    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
+    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
+    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
+    62924, 63050, 63082, 63274, 63390, 
  };
-#define HIBIT ((unsigned char)0x80)
-  unsigned char aChar[] = {
-    '\0',      'a',       'c',       'e',       'i',       'n',       
-    'o',       'u',       'y',       'y',       'a',       'c',       
-    'd',       'e',       'e',       'g',       'h',       'i',       
-    'j',       'k',       'l',       'n',       'o',       'r',       
-    's',       't',       'u',       'u',       'w',       'y',       
-    'z',       'o',       'u',       'a',       'i',       'o',       
-    'u',       'u'|HIBIT, 'a'|HIBIT, 'g',       'k',       'o',       
-    'o'|HIBIT, 'j',       'g',       'n',       'a'|HIBIT, 'a',       
-    'e',       'i',       'o',       'r',       'u',       's',       
-    't',       'h',       'a',       'e',       'o'|HIBIT, 'o',       
-    'o'|HIBIT, 'y',       '\0',      '\0',      '\0',      '\0',      
-    '\0',      '\0',      '\0',      '\0',      'a',       'b',       
-    'c'|HIBIT, 'd',       'd',       'e'|HIBIT, 'e',       'e'|HIBIT, 
-    'f',       'g',       'h',       'h',       'i',       'i'|HIBIT, 
-    'k',       'l',       'l'|HIBIT, 'l',       'm',       'n',       
-    'o'|HIBIT, 'p',       'r',       'r'|HIBIT, 'r',       's',       
-    's'|HIBIT, 't',       'u',       'u'|HIBIT, 'v',       'w',       
-    'w',       'x',       'y',       'z',       'h',       't',       
-    'w',       'y',       'a',       'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 
-    'e',       'e'|HIBIT, 'e'|HIBIT, 'i',       'o',       'o'|HIBIT, 
-    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',       
+  char aChar[] = {
+    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
+    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
+    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
+    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
+    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
+    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
+    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
+    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
+    'e',  'i',  'o',  'u',  'y',  
  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -217,8 +201,7 @@ static int remove_diacritic(int c, int bComplex){
    }
  }
  assert( key>=aDia[iRes] );
-  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
-  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
+  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
 }


@@ -231,8 +214,8 @@ int sqlite3FtsUnicodeIsdiacritic(int c){
  unsigned int mask1 = 0x000361F8;
  if( c<768 || c>817 ) return 0;
  return (c < 768+32) ?
-      (mask0 & ((unsigned int)1 << (c-768))) :
-      (mask1 & ((unsigned int)1 << (c-768-32)));
+      (mask0 & (1 << (c-768))) :
+      (mask1 & (1 << (c-768-32)));
 }


@@ -245,7 +228,7 @@ int sqlite3FtsUnicodeIsdiacritic(int c){
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
+int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
@@ -339,17 +322,16 @@ int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){

  int ret = c;

+  assert( c>=0 );
  assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );

  if( c<128 ){
    if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
  }else if( c<65536 ){
-    const struct TableEntry *p;
    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
    int iLo = 0;
    int iRes = -1;

-    assert( c>aEntry[0].iCode );
    while( iHi>=iLo ){
      int iTest = (iHi + iLo) / 2;
      int cmp = (c - aEntry[iTest].iCode);
@@ -360,17 +342,17 @@ int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
        iHi = iTest-1;
      }
    }
+    assert( iRes<0 || c>=aEntry[iRes].iCode );

-    assert( iRes>=0 && c>=aEntry[iRes].iCode );
-    p = &aEntry[iRes];
-    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
-      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
-      assert( ret>0 );
+    if( iRes>=0 ){
+      const struct TableEntry *p = &aEntry[iRes];
+      if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
+        ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
+        assert( ret>0 );
+      }
    }

-    if( eRemoveDiacritic ){
-      ret = remove_diacritic(ret, eRemoveDiacritic==2);
-    }
+    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
  }
  
  else if( c>=66560 && c<66600 ){
@@ -23,7 +23,7 @@
 #include <string.h>
 #include <assert.h>
 #include <stdlib.h>
-#include <stdio.h>
+

 #define FTS_MAX_APPENDABLE_HEIGHT 16

@@ -67,7 +67,7 @@ int test_fts3_node_chunk_threshold = (4*1024)*4;
 #endif

 /*
-** The values that may be meaningfully bound to the :1 parameter in
+** The two values that may be meaningfully bound to the :1 parameter in
 ** statements SQL_REPLACE_STAT and SQL_SELECT_STAT.
 */
 #define FTS_STAT_DOCTOTAL      0
@@ -335,7 +335,7 @@ static int fts3SqlStmt(
 ** returns zero rows.  */
 /* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' "
         "  GROUP BY level HAVING cnt>=?"
-         "  ORDER BY (level %% 1024) ASC, 2 DESC LIMIT 1",
+         "  ORDER BY (level %% 1024) ASC LIMIT 1",

 /* Estimate the upper limit on the number of leaf nodes in a new segment
 ** created by merging the oldest :2 segments from absolute level :1. See 
@@ -396,12 +396,10 @@ static int fts3SqlStmt(
  
  pStmt = p->aStmt[eStmt];
  if( !pStmt ){
-    int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB;
    char *zSql;
    if( eStmt==SQL_CONTENT_INSERT ){
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
    }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
-      f &= ~SQLITE_PREPARE_NO_VTAB;
      zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
    }else{
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
@@ -409,7 +407,7 @@ static int fts3SqlStmt(
    if( !zSql ){
      rc = SQLITE_NOMEM;
    }else{
-      rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL);
+      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
      sqlite3_free(zSql);
      assert( rc==SQLITE_OK || pStmt==0 );
      p->aStmt[eStmt] = pStmt;
@@ -567,7 +565,7 @@ static sqlite3_int64 getAbsoluteLevel(
  int iLevel                      /* Level of segments */
 ){
  sqlite3_int64 iBase;            /* First absolute level for iLangid/iIndex */
-  assert_fts3_nc( iLangid>=0 );
+  assert( iLangid>=0 );
  assert( p->nIndex>0 );
  assert( iIndex>=0 && iIndex<p->nIndex );

@@ -696,7 +694,7 @@ static int fts3PendingListAppend(
  assert( !p || p->iLastDocid<=iDocid );

  if( !p || p->iLastDocid!=iDocid ){
-    u64 iDelta = (u64)iDocid - (u64)(p ? p->iLastDocid : 0);
+    sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
    if( p ){
      assert( p->nData<p->nSpace );
      assert( p->aData[p->nData]==0 );
@@ -1153,7 +1151,7 @@ static int fts3AllocateSegdirIdx(
    ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
    ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
    */
-    if( iNext>=MergeCount(p) ){
+    if( iNext>=FTS3_MERGE_COUNT ){
      fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel));
      rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel);
      *piIdx = 0;
@@ -1237,8 +1235,6 @@ int sqlite3Fts3ReadBlock(
      }
      *paBlob = aByte;
    }
-  }else if( rc==SQLITE_ERROR ){
-    rc = FTS_CORRUPT_VTAB;
  }

  return rc;
@@ -1350,9 +1346,7 @@ static int fts3SegReaderNext(

    /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf 
    ** blocks have already been traversed.  */
-#ifdef CORRUPT_DB
-    assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock || CORRUPT_DB );
-#endif
+    assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
    if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
      return SQLITE_OK;
    }
@@ -1379,19 +1373,15 @@ static int fts3SegReaderNext(
  ** safe (no risk of overread) even if the node data is corrupted. */
  pNext += fts3GetVarint32(pNext, &nPrefix);
  pNext += fts3GetVarint32(pNext, &nSuffix);
-  if( nSuffix<=0 
-   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
-   || nPrefix>pReader->nTerm
+  if( nPrefix<0 || nSuffix<=0 
+   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
  ){
    return FTS_CORRUPT_VTAB;
  }

-  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
-  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
-  ** overflow - hence the (i64) casts.  */
-  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
-    i64 nNew = ((i64)nPrefix+nSuffix)*2;
-    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
+  if( nPrefix+nSuffix>pReader->nTermAlloc ){
+    int nNew = (nPrefix+nSuffix)*2;
+    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
@@ -1413,9 +1403,8 @@ static int fts3SegReaderNext(
  ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  ** of these statements is untrue, then the data structure is corrupt.
  */
-  if( pReader->nDoclist > pReader->nNode-(pReader->aDoclist-pReader->aNode)
+  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
   || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
-   || pReader->nDoclist==0
  ){
    return FTS_CORRUPT_VTAB;
  }
@@ -1532,18 +1521,18 @@ static int fts3SegReaderNextDocid(
    }else{
      rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
      if( rc==SQLITE_OK ){
-        u64 iDelta;
-        pReader->pOffsetList = p + sqlite3Fts3GetVarintU(p, &iDelta);
+        sqlite3_int64 iDelta;
+        pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
        if( pTab->bDescIdx ){
-          pReader->iDocid = (i64)((u64)pReader->iDocid - iDelta);
+          pReader->iDocid -= iDelta;
        }else{
-          pReader->iDocid = (i64)((u64)pReader->iDocid + iDelta);
+          pReader->iDocid += iDelta;
        }
      }
    }
  }

-  return rc;
+  return SQLITE_OK;
 }


@@ -1614,13 +1603,8 @@ int sqlite3Fts3SegReaderNew(
  Fts3SegReader *pReader;         /* Newly allocated SegReader object */
  int nExtra = 0;                 /* Bytes to allocate segment root node */

-  assert( zRoot!=0 || nRoot==0 );
-#ifdef CORRUPT_DB
-  assert( zRoot!=0 || CORRUPT_DB );
-#endif
-
+  assert( iStartLeaf<=iEndLeaf );
  if( iStartLeaf==0 ){
-    if( iEndLeaf!=0 ) return FTS_CORRUPT_VTAB;
    nExtra = nRoot + FTS3_NODE_PADDING;
  }

@@ -1640,7 +1624,7 @@ int sqlite3Fts3SegReaderNew(
    pReader->aNode = (char *)&pReader[1];
    pReader->rootOnly = 1;
    pReader->nNode = nRoot;
-    if( nRoot ) memcpy(pReader->aNode, zRoot, nRoot);
+    memcpy(pReader->aNode, zRoot, nRoot);
    memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING);
  }else{
    pReader->iCurrentBlock = iStartLeaf-1;
@@ -1755,9 +1739,8 @@ int sqlite3Fts3SegReaderPending(
  }

  if( nElem>0 ){
-    sqlite3_int64 nByte;
-    nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *);
-    pReader = (Fts3SegReader *)sqlite3_malloc64(nByte);
+    int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *);
+    pReader = (Fts3SegReader *)sqlite3_malloc(nByte);
    if( !pReader ){
      rc = SQLITE_NOMEM;
    }else{
@@ -1924,7 +1907,6 @@ static int fts3WriteSegment(
    sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
    sqlite3_step(pStmt);
    rc = sqlite3_reset(pStmt);
-    sqlite3_bind_null(pStmt, 2);
  }
  return rc;
 }
@@ -1981,7 +1963,6 @@ static int fts3WriteSegdir(
    sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
    sqlite3_step(pStmt);
    rc = sqlite3_reset(pStmt);
-    sqlite3_bind_null(pStmt, 6);
  }
  return rc;
 }
@@ -2033,11 +2014,6 @@ static int fts3NodeAddTerm(
    nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm);
    nSuffix = nTerm-nPrefix;

-    /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of 
-    ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when
-    ** compared with BINARY collation. This indicates corruption.  */
-    if( nSuffix<=0 ) return FTS_CORRUPT_VTAB;
-
    nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix;
    if( nReq<=p->nNodeSize || !pTree->zTerm ){

@@ -2266,11 +2242,6 @@ static int fts3SegWriterAdd(
  nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm);
  nSuffix = nTerm-nPrefix;

-  /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of 
-  ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when
-  ** compared with BINARY collation. This indicates corruption.  */
-  if( nSuffix<=0 ) return FTS_CORRUPT_VTAB;
-
  /* Figure out how many bytes are required by this new entry */
  nReq = sqlite3Fts3VarintLen(nPrefix) +    /* varint containing prefix size */
    sqlite3Fts3VarintLen(nSuffix) +         /* varint containing suffix size */
@@ -2282,7 +2253,6 @@ static int fts3SegWriterAdd(
    int rc;

    /* The current leaf node is full. Write it out to the database. */
-    if( pWriter->iFree==LARGEST_INT64 ) return FTS_CORRUPT_VTAB;
    rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
    if( rc!=SQLITE_OK ) return rc;
    p->nLeafAdd++;
@@ -2332,11 +2302,9 @@ static int fts3SegWriterAdd(
  /* Append the prefix-compressed term and doclist to the buffer. */
  nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix);
  nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix);
-  assert( nSuffix>0 );
  memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix);
  nData += nSuffix;
  nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist);
-  assert( nDoclist>0 );
  memcpy(&pWriter->aData[nData], aDoclist, nDoclist);
  pWriter->nData = nData + nDoclist;

@@ -2356,7 +2324,6 @@ static int fts3SegWriterAdd(
      pWriter->zTerm = zNew;
    }
    assert( pWriter->zTerm==pWriter->zMalloc );
-    assert( nTerm>0 );
    memcpy(pWriter->zTerm, zTerm, nTerm);
  }else{
    pWriter->zTerm = (char *)zTerm;
@@ -2503,7 +2470,7 @@ static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){
  if( rc!=SQLITE_OK ) return rc;
  sqlite3_bind_int64(pStmt, 1, iAbsLevel+1);
  sqlite3_bind_int64(pStmt, 2, 
-      (((u64)iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL
+      ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL
  );

  *pbMax = 0;
@@ -2631,14 +2598,14 @@ static void fts3ColumnFilter(

    nList -= (int)(p - pList);
    pList = p;
-    if( nList<=0 ){
+    if( nList==0 ){
      break;
    }
    p = &pList[1];
    p += fts3GetVarint32(p, &iCurrent);
  }

-  if( bZero && (pEnd - &pList[nList])>0){
+  if( bZero && &pList[nList]!=pEnd ){
    memset(&pList[nList], 0, pEnd - &pList[nList]);
  }
  *ppList = pList;
@@ -2665,7 +2632,6 @@ static int fts3MsrBufferData(
    pMsr->aBuffer = pNew;
  }

-  assert( nList>0 );
  memcpy(pMsr->aBuffer, pList, nList);
  return SQLITE_OK;
 }
@@ -2979,12 +2945,12 @@ int sqlite3Fts3SegReaderStep(
          ** doclist. */
          sqlite3_int64 iDelta;
          if( p->bDescIdx && nDoclist>0 ){
-            if( iPrev<=iDocid ) return FTS_CORRUPT_VTAB;
-            iDelta = (i64)((u64)iPrev - (u64)iDocid);
+            iDelta = iPrev - iDocid;
          }else{
-            if( nDoclist>0 && iPrev>=iDocid ) return FTS_CORRUPT_VTAB;
-            iDelta = (i64)((u64)iDocid - (u64)iPrev);
+            iDelta = iDocid - iPrev;
          }
+          assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) );
+          assert( nDoclist>0 || iDelta==iDocid );

          nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
          if( nDoclist+nByte>pCsr->nBuffer ){
@@ -3069,11 +3035,11 @@ static void fts3ReadEndBlockField(
  if( zText ){
    int i;
    int iMul = 1;
-    u64 iVal = 0;
+    i64 iVal = 0;
    for(i=0; zText[i]>='0' && zText[i]<='9'; i++){
      iVal = iVal*10 + (zText[i] - '0');
    }
-    *piEndBlock = (i64)iVal;
+    *piEndBlock = iVal;
    while( zText[i]==' ' ) i++;
    iVal = 0;
    if( zText[i]=='-' ){
@@ -3083,7 +3049,7 @@ static void fts3ReadEndBlockField(
    for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){
      iVal = iVal*10 + (zText[i] - '0');
    }
-    *pnByte = ((i64)iVal * (i64)iMul);
+    *pnByte = (iVal * (i64)iMul);
  }
 }

@@ -3249,10 +3215,8 @@ static int fts3SegmentMerge(
  if( rc!=SQLITE_OK ) goto finished;

  assert( csr.nSegment>0 );
-  assert_fts3_nc( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
-  assert_fts3_nc( 
-    iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) 
-  );
+  assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
+  assert( iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) );

  memset(&filter, 0, sizeof(Fts3SegFilter));
  filter.flags = FTS3_SEGMENT_REQUIRE_POS;
@@ -3266,7 +3230,7 @@ static int fts3SegmentMerge(
        csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
  }
  if( rc!=SQLITE_OK ) goto finished;
-  assert_fts3_nc( pWriter || bIgnoreEmpty );
+  assert( pWriter || bIgnoreEmpty );

  if( iLevel!=FTS3_SEGCURSOR_PENDING ){
    rc = fts3DeleteSegdir(
@@ -3351,16 +3315,14 @@ static void fts3DecodeIntArray(
  const char *zBuf,  /* The BLOB containing the varints */
  int nBuf           /* size of the BLOB */
 ){
-  int i = 0;
-  if( nBuf && (zBuf[nBuf-1]&0x80)==0 ){
-    int j;
-    for(i=j=0; i<N && j<nBuf; i++){
-      sqlite3_int64 x;
-      j += sqlite3Fts3GetVarint(&zBuf[j], &x);
-      a[i] = (u32)(x & 0xffffffff);
-    }
+  int i, j;
+  UNUSED_PARAMETER(nBuf);
+  for(i=j=0; i<N; i++){
+    sqlite3_int64 x;
+    j += sqlite3Fts3GetVarint(&zBuf[j], &x);
+    assert(j<=nBuf);
+    a[i] = (u32)(x & 0xffffffff);
  }
-  while( i<N ) a[i++] = 0;
 }

 /*
@@ -3379,7 +3341,7 @@ static void fts3InsertDocsize(
  int rc;                  /* Result code from subfunctions */

  if( *pRC ) return;
-  pBlob = sqlite3_malloc64( 10*(sqlite3_int64)p->nColumn );
+  pBlob = sqlite3_malloc( 10*p->nColumn );
  if( pBlob==0 ){
    *pRC = SQLITE_NOMEM;
    return;
@@ -3429,7 +3391,7 @@ static void fts3UpdateDocTotals(
  const int nStat = p->nColumn+2;

  if( *pRC ) return;
-  a = sqlite3_malloc64( (sizeof(u32)+10)*(sqlite3_int64)nStat );
+  a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
  if( a==0 ){
    *pRC = SQLITE_NOMEM;
    return;
@@ -3480,7 +3442,6 @@ static void fts3UpdateDocTotals(
  sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
  sqlite3_step(pStmt);
  *pRC = sqlite3_reset(pStmt);
-  sqlite3_bind_null(pStmt, 2);
  sqlite3_free(a);
 }

@@ -3493,10 +3454,7 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
  int rc;
  sqlite3_stmt *pAllLangid = 0;

-  rc = sqlite3Fts3PendingTermsFlush(p);
-  if( rc==SQLITE_OK ){
-    rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
-  }
+  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
  if( rc==SQLITE_OK ){
    int rc2;
    sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
@@ -3517,6 +3475,7 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
  }

  sqlite3Fts3SegmentsClose(p);
+  sqlite3Fts3PendingTermsClear(p);

  return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
 }
@@ -3552,8 +3511,8 @@ static int fts3DoRebuild(Fts3Table *p){
    }

    if( rc==SQLITE_OK ){
-      sqlite3_int64 nByte = sizeof(u32) * ((sqlite3_int64)p->nColumn+1)*3;
-      aSz = (u32 *)sqlite3_malloc64(nByte);
+      int nByte = sizeof(u32) * (p->nColumn+1)*3;
+      aSz = (u32 *)sqlite3_malloc(nByte);
      if( aSz==0 ){
        rc = SQLITE_NOMEM;
      }else{
@@ -3619,12 +3578,12 @@ static int fts3IncrmergeCsr(
 ){
  int rc;                         /* Return Code */
  sqlite3_stmt *pStmt = 0;        /* Statement used to read %_segdir entry */  
-  sqlite3_int64 nByte;            /* Bytes allocated at pCsr->apSegment[] */
+  int nByte;                      /* Bytes allocated at pCsr->apSegment[] */

  /* Allocate space for the Fts3MultiSegReader.aCsr[] array */
  memset(pCsr, 0, sizeof(*pCsr));
  nByte = sizeof(Fts3SegReader *) * nSeg;
-  pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc64(nByte);
+  pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);

  if( pCsr->apSegment==0 ){
    rc = SQLITE_NOMEM;
@@ -3767,9 +3726,6 @@ static int nodeReaderNext(NodeReader *p){
    }
    p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);

-    if( nPrefix>p->term.n || nSuffix>p->nNode-p->iOff || nSuffix==0 ){
-      return FTS_CORRUPT_VTAB;
-    }
    blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
    if( rc==SQLITE_OK ){
      memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
@@ -3777,16 +3733,14 @@ static int nodeReaderNext(NodeReader *p){
      p->iOff += nSuffix;
      if( p->iChild==0 ){
        p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
-        if( (p->nNode-p->iOff)<p->nDoclist ){
-          return FTS_CORRUPT_VTAB;
-        }
        p->aDoclist = &p->aNode[p->iOff];
        p->iOff += p->nDoclist;
      }
    }
  }

-  assert_fts3_nc( p->iOff<=p->nNode );
+  assert( p->iOff<=p->nNode );
+
  return rc;
 }

@@ -3810,14 +3764,14 @@ static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){
  p->nNode = nNode;

  /* Figure out if this is a leaf or an internal node. */
-  if( aNode && aNode[0] ){
+  if( p->aNode[0] ){
    /* An internal node. */
    p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild);
  }else{
    p->iOff = 1;
  }

-  return aNode ? nodeReaderNext(p) : SQLITE_OK;
+  return nodeReaderNext(p);
 }

 /*
@@ -3854,7 +3808,6 @@ static int fts3IncrmergePush(
    ** be added to.  */
    nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm);
    nSuffix = nTerm - nPrefix;
-    if(nSuffix<=0 ) return FTS_CORRUPT_VTAB;
    nSpace  = sqlite3Fts3VarintLen(nPrefix);
    nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;

@@ -3948,14 +3901,13 @@ static int fts3AppendToNode(
  /* Node must have already been started. There must be a doclist for a
  ** leaf node, and there must not be a doclist for an internal node.  */
  assert( pNode->n>0 );
-  assert_fts3_nc( (pNode->a[0]=='\0')==(aDoclist!=0) );
+  assert( (pNode->a[0]=='\0')==(aDoclist!=0) );

  blobGrowBuffer(pPrev, nTerm, &rc);
  if( rc!=SQLITE_OK ) return rc;

  nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm);
  nSuffix = nTerm - nPrefix;
-  if( nSuffix<=0 ) return FTS_CORRUPT_VTAB;
  memcpy(pPrev->a, zTerm, nTerm);
  pPrev->n = nTerm;

@@ -4165,7 +4117,7 @@ static int fts3TermCmp(
  int nCmp = MIN(nLhs, nRhs);
  int res;

-  res = (nCmp ? memcmp(zLhs, zRhs, nCmp) : 0);
+  res = memcmp(zLhs, zRhs, nCmp);
  if( res==0 ) res = nLhs - nRhs;

  return res;
@@ -4249,10 +4201,6 @@ static int fts3IncrmergeLoad(
      pWriter->bNoLeafData = (pWriter->nLeafData==0);
      nRoot = sqlite3_column_bytes(pSelect, 4);
      aRoot = sqlite3_column_blob(pSelect, 4);
-      if( aRoot==0 ){
-        sqlite3_reset(pSelect);
-        return nRoot ? SQLITE_NOMEM : FTS_CORRUPT_VTAB;
-      }
    }else{
      return sqlite3_reset(pSelect);
    }
@@ -4288,10 +4236,6 @@ static int fts3IncrmergeLoad(
      int i;
      int nHeight = (int)aRoot[0];
      NodeWriter *pNode;
-      if( nHeight<1 || nHeight>FTS_MAX_APPENDABLE_HEIGHT ){
-        sqlite3_reset(pSelect);
-        return FTS_CORRUPT_VTAB;
-      }

      pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT;
      pWriter->iStart = iStart;
@@ -4305,42 +4249,34 @@ static int fts3IncrmergeLoad(

      pNode = &pWriter->aNodeWriter[nHeight];
      pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight;
-      blobGrowBuffer(&pNode->block, 
-          MAX(nRoot, p->nNodeSize)+FTS3_NODE_PADDING, &rc
-      );
+      blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc);
      if( rc==SQLITE_OK ){
        memcpy(pNode->block.a, aRoot, nRoot);
        pNode->block.n = nRoot;
-        memset(&pNode->block.a[nRoot], 0, FTS3_NODE_PADDING);
      }

      for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){
        NodeReader reader;
        pNode = &pWriter->aNodeWriter[i];

-        if( pNode->block.a){
-          rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n);
-          while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader);
-          blobGrowBuffer(&pNode->key, reader.term.n, &rc);
-          if( rc==SQLITE_OK ){
-            memcpy(pNode->key.a, reader.term.a, reader.term.n);
-            pNode->key.n = reader.term.n;
-            if( i>0 ){
-              char *aBlock = 0;
-              int nBlock = 0;
-              pNode = &pWriter->aNodeWriter[i-1];
-              pNode->iBlock = reader.iChild;
-              rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0);
-              blobGrowBuffer(&pNode->block, 
-                  MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc
-              );
-              if( rc==SQLITE_OK ){
-                memcpy(pNode->block.a, aBlock, nBlock);
-                pNode->block.n = nBlock;
-                memset(&pNode->block.a[nBlock], 0, FTS3_NODE_PADDING);
-              }
-              sqlite3_free(aBlock);
+        rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n);
+        while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader);
+        blobGrowBuffer(&pNode->key, reader.term.n, &rc);
+        if( rc==SQLITE_OK ){
+          memcpy(pNode->key.a, reader.term.a, reader.term.n);
+          pNode->key.n = reader.term.n;
+          if( i>0 ){
+            char *aBlock = 0;
+            int nBlock = 0;
+            pNode = &pWriter->aNodeWriter[i-1];
+            pNode->iBlock = reader.iChild;
+            rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0);
+            blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc);
+            if( rc==SQLITE_OK ){
+              memcpy(pNode->block.a, aBlock, nBlock);
+              pNode->block.n = nBlock;
            }
+            sqlite3_free(aBlock);
          }
        }
        nodeReaderRelease(&reader);
@@ -4583,10 +4519,7 @@ static int fts3TruncateNode(
  NodeReader reader;              /* Reader object */
  Blob prev = {0, 0, 0};          /* Previous term written to new node */
  int rc = SQLITE_OK;             /* Return code */
-  int bLeaf;                       /* True for a leaf node */
-
-  if( nNode<1 ) return FTS_CORRUPT_VTAB;
-  bLeaf = aNode[0]=='\0';
+  int bLeaf = aNode[0]=='\0';     /* True for a leaf node */

  /* Allocate required output space */
  blobGrowBuffer(pNew, nNode, &rc);
@@ -4697,7 +4630,6 @@ static int fts3TruncateSegment(
      sqlite3_bind_int(pChomp, 4, iIdx);
      sqlite3_step(pChomp);
      rc = sqlite3_reset(pChomp);
-      sqlite3_bind_null(pChomp, 2);
    }
  }

@@ -4777,7 +4709,6 @@ static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){
    sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
-    sqlite3_bind_null(pReplace, 2);
  }

  return rc;
@@ -4852,17 +4783,13 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){
  const int nHint = pHint->n;
  int i;

-  i = pHint->n-1;
-  if( (pHint->a[i] & 0x80) ) return FTS_CORRUPT_VTAB;
+  i = pHint->n-2;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
-  if( i==0 ) return FTS_CORRUPT_VTAB;
-  i--;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;

  pHint->n = i;
  i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
  i += fts3GetVarint32(&pHint->a[i], pnInput);
-  assert( i<=nHint );
  if( i!=nHint ) return FTS_CORRUPT_VTAB;

  return SQLITE_OK;
@@ -4932,14 +4859,8 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){

      rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg);
      if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){
-        /* Based on the scan in the block above, it is known that there
-        ** are no levels with a relative level smaller than that of
-        ** iAbsLevel with more than nSeg segments, or if nSeg is -1, 
-        ** no levels with more than nMin segments. Use this to limit the
-        ** value of nHintSeg to avoid a large memory allocation in case the 
-        ** merge-hint is corrupt*/
        iAbsLevel = iHintAbsLevel;
-        nSeg = MIN(MAX(nMin,nSeg), nHintSeg);
+        nSeg = nHintSeg;
        bUseHint = 1;
        bDirtyHint = 1;
      }else{
@@ -4952,13 +4873,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
    /* If nSeg is less that zero, then there is no level with at least
    ** nMin segments and no hint in the %_stat table. No work to do.
    ** Exit early in this case.  */
-    if( nSeg<=0 ) break;
-
-    assert( nMod<=0x7FFFFFFF );
-    if( iAbsLevel<0 || iAbsLevel>(nMod<<32) ){
-      rc = FTS_CORRUPT_VTAB;
-      break;
-    }
+    if( nSeg<0 ) break;

    /* Open a cursor to iterate through the contents of the oldest nSeg 
    ** indexes of absolute level iAbsLevel. If this cursor is opened using 
@@ -4986,15 +4901,8 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
    }
    if( SQLITE_OK==rc && pCsr->nSegment==nSeg
     && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
+     && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
    ){
-      int bEmpty = 0;
-      rc = sqlite3Fts3SegReaderStep(p, pCsr);
-      if( rc==SQLITE_OK ){
-        bEmpty = 1;
-      }else if( rc!=SQLITE_ROW ){
-        sqlite3Fts3SegReaderFinish(pCsr);
-        break;
-      }
      if( bUseHint && iIdx>0 ){
        const char *zKey = pCsr->zTerm;
        int nKey = pCsr->nTerm;
@@ -5005,13 +4913,11 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){

      if( rc==SQLITE_OK && pWriter->nLeafEst ){
        fts3LogMerge(nSeg, iAbsLevel);
-        if( bEmpty==0 ){
-          do {
-            rc = fts3IncrmergeAppend(p, pWriter, pCsr);
-            if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
-            if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
-          }while( rc==SQLITE_ROW );
-        }
+        do {
+          rc = fts3IncrmergeAppend(p, pWriter, pCsr);
+          if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
+          if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
+        }while( rc==SQLITE_ROW );

        /* Update or delete the input segments */
        if( rc==SQLITE_OK ){
@@ -5050,14 +4956,11 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
 ** Convert the text beginning at *pz into an integer and return
 ** its value.  Advance *pz to point to the first character past
 ** the integer.
-**
-** This function used for parameters to merge= and incrmerge=
-** commands. 
 */
 static int fts3Getint(const char **pz){
  const char *z = *pz;
  int i = 0;
-  while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0';
+  while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0';
  *pz = z;
  return i;
 }
@@ -5076,7 +4979,7 @@ static int fts3DoIncrmerge(
  const char *zParam              /* Nul-terminated string containing "A,B" */
 ){
  int rc;
-  int nMin = (MergeCount(p) / 2);
+  int nMin = (FTS3_MERGE_COUNT / 2);
  int nMerge = 0;
  const char *z = zParam;

@@ -5121,7 +5024,7 @@ static int fts3DoAutoincrmerge(
  int rc = SQLITE_OK;
  sqlite3_stmt *pStmt = 0;
  p->nAutoincrmerge = fts3Getint(&zParam);
-  if( p->nAutoincrmerge==1 || p->nAutoincrmerge>MergeCount(p) ){
+  if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){
    p->nAutoincrmerge = 8;
  }
  if( !p->bHasStat ){
@@ -5204,12 +5107,12 @@ static u64 fts3ChecksumIndex(

      i64 iDocid = 0;
      i64 iCol = 0;
-      u64 iPos = 0;
+      i64 iPos = 0;

      pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid);
      while( pCsr<pEnd ){
-        u64 iVal = 0;
-        pCsr += sqlite3Fts3GetVarintU(pCsr, &iVal);
+        i64 iVal = 0;
+        pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
        if( pCsr<pEnd ){
          if( iVal==0 || iVal==1 ){
            iCol = 0;
@@ -5217,12 +5120,8 @@ static u64 fts3ChecksumIndex(
            if( iVal ){
              pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
            }else{
-              pCsr += sqlite3Fts3GetVarintU(pCsr, &iVal);
-              if( p->bDescIdx ){
-                iDocid = (i64)((u64)iDocid - iVal);
-              }else{
-                iDocid = (i64)((u64)iDocid + iVal);
-              }
+              pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
+              iDocid += iVal;
            }
          }else{
            iPos += (iVal - 2);
@@ -5295,9 +5194,10 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
      for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
        if( p->abNotindexed[iCol]==0 ){
          const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1);
+          int nText = sqlite3_column_bytes(pStmt, iCol+1);
          sqlite3_tokenizer_cursor *pT = 0;

-          rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, -1, &pT);
+          rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT);
          while( rc==SQLITE_OK ){
            char const *zToken;       /* Buffer containing token */
            int nToken = 0;           /* Number of bytes in token */
@@ -5382,7 +5282,7 @@ static int fts3DoIntegrityCheck(
 ** meaningful value to insert is the text 'optimize'.
 */
 static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
-  int rc = SQLITE_ERROR;           /* Return Code */
+  int rc;                         /* Return Code */
  const char *zVal = (const char *)sqlite3_value_text(pVal);
  int nVal = sqlite3_value_bytes(pVal);

@@ -5398,27 +5298,21 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
    rc = fts3DoIncrmerge(p, &zVal[6]);
  }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
    rc = fts3DoAutoincrmerge(p, &zVal[10]);
-#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
-  }else{
-    int v;
-    if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
-      v = atoi(&zVal[9]);
-      if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v;
-      rc = SQLITE_OK;
-    }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
-      v = atoi(&zVal[11]);
-      if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v;
-      rc = SQLITE_OK;
-    }else if( nVal>21 && 0==sqlite3_strnicmp(zVal,"test-no-incr-doclist=",21) ){
-      p->bNoIncrDoclist = atoi(&zVal[21]);
-      rc = SQLITE_OK;
-    }else if( nVal>11 && 0==sqlite3_strnicmp(zVal,"mergecount=",11) ){
-      v = atoi(&zVal[11]);
-      if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v;
-      rc = SQLITE_OK;
-    }
+#ifdef SQLITE_TEST
+  }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
+    p->nNodeSize = atoi(&zVal[9]);
+    rc = SQLITE_OK;
+  }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
+    p->nMaxPendingData = atoi(&zVal[11]);
+    rc = SQLITE_OK;
+  }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){
+    p->bNoIncrDoclist = atoi(&zVal[21]);
+    rc = SQLITE_OK;
 #endif
+  }else{
+    rc = SQLITE_ERROR;
  }
+
  return rc;
 }

@@ -5626,6 +5520,7 @@ int sqlite3Fts3UpdateMethod(
 ){
  Fts3Table *p = (Fts3Table *)pVtab;
  int rc = SQLITE_OK;             /* Return Code */
+  int isRemove = 0;               /* True for an UPDATE or DELETE */
  u32 *aSzIns = 0;                /* Sizes of inserted documents */
  u32 *aSzDel = 0;                /* Sizes of deleted documents */
  int nChng = 0;                  /* Net change in number of documents */
@@ -5659,7 +5554,7 @@ int sqlite3Fts3UpdateMethod(
  }

  /* Allocate space to hold the change in document sizes */
-  aSzDel = sqlite3_malloc64(sizeof(aSzDel[0])*((sqlite3_int64)p->nColumn+1)*2);
+  aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 );
  if( aSzDel==0 ){
    rc = SQLITE_NOMEM;
    goto update_out;
@@ -5723,6 +5618,7 @@ int sqlite3Fts3UpdateMethod(
  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
    rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
+    isRemove = 1;
  }
  
  /* If this is an INSERT or UPDATE operation, insert the new record. */
@@ -5734,7 +5630,7 @@ int sqlite3Fts3UpdateMethod(
        rc = FTS_CORRUPT_VTAB;
      }
    }
-    if( rc==SQLITE_OK ){
+    if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
      rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
    }
    if( rc==SQLITE_OK ){
@@ -1,16 +0,0 @@
-#!/bin/sh
-
-set -e
-
-srcdir=`dirname $(dirname $(dirname $(dirname $0)))`
-./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt
-
-echo ""
-
-for f in `ls $srcdir/ext/fts3/*.c` 
-do
-  f=`basename $f`
-  echo -ne "$f: "
-  gcov -b $f | grep Taken | sed 's/Taken at least once://'
-done
-
@@ -9,12 +9,11 @@ proc print_rd {map} {
  set nRange 1
  set iFirst  [lindex $map 0 0]
  set cPrev   [lindex $map 0 1]
-  set fPrev   [lindex $map 0 2]

  foreach m [lrange $map 1 end] {
-    foreach {i c f} $m {}
+    foreach {i c} $m {}

-    if {$cPrev == $c && $fPrev==$f} {
+    if {$cPrev == $c} {
      for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
        if {[info exists tl_lookup_table($j)]==0} break
      }
@@ -30,16 +29,13 @@ proc print_rd {map} {

    lappend lRange [list $iFirst $nRange]
    lappend aChar  $cPrev
-    lappend aFlag  $fPrev

    set iFirst $i
    set cPrev  $c
-    set fPrev  $f
    set nRange 1
  }
  lappend lRange [list $iFirst $nRange]
  lappend aChar $cPrev
-  lappend aFlag $fPrev

  puts "/*"
  puts "** If the argument is a codepoint corresponding to a lowercase letter"
@@ -49,7 +45,7 @@ proc print_rd {map} {
  puts "** E\"). The resuls of passing a codepoint that corresponds to an"
  puts "** uppercase letter are undefined."
  puts "*/"
-  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
+  puts "static int ${::remove_diacritic}(int c)\{"
  puts "  unsigned short aDia\[\] = \{"
  puts -nonewline "        0, "
  set i 1
@@ -63,19 +59,14 @@ proc print_rd {map} {
  }
  puts ""
  puts "  \};"
-  puts "#define HIBIT ((unsigned char)0x80)"
-  puts "  unsigned char aChar\[\] = \{"
-  puts -nonewline "    '\\0',      "
+  puts "  char aChar\[\] = \{"
+  puts -nonewline "    '\\0', "
  set i 1
-  foreach c $aChar f $aFlag {
-    if { $f } {
-      set str "'$c'|HIBIT, "
-    } else {
-      set str "'$c',       "
-    }
-    if {$c == ""} { set str "'\\0',      " }
+  foreach c $aChar {
+    set str "'$c',  "
+    if {$c == ""} { set str "'\\0', " }

-    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
+    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
    incr i
    puts -nonewline "$str"
  }
@@ -96,8 +87,7 @@ proc print_rd {map} {
    }
  }
  assert( key>=aDia[iRes] );
-  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
-  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
+  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
  puts "\}"
 }

@@ -105,8 +95,7 @@ proc print_isdiacritic {zFunc map} {

  set lCode [list]
  foreach m $map {
-    foreach {code char flag} $m {}
-    if {$flag} continue
+    foreach {code char} $m {}
    if {$code && $char == ""} { lappend lCode $code }
  }
  set lCode [lsort -integer $lCode]
@@ -135,8 +124,8 @@ proc print_isdiacritic {zFunc map} {

  puts "  if( c<$iFirst || c>$iLast ) return 0;"
  puts "  return (c < $iFirst+32) ?"
-  puts "      (mask0 & ((unsigned int)1 << (c-$iFirst))) :"
-  puts "      (mask1 & ((unsigned int)1 << (c-$iFirst-32)));"
+  puts "      (mask0 & (1 << (c-$iFirst))) :"
+  puts "      (mask1 & (1 << (c-$iFirst-32)));"
  puts "\}"
 }

@@ -238,7 +227,7 @@ proc print_isalnum {zFunc lRange} {
  an_print_ascii_bitmap $lRange
  puts {
  if( (unsigned int)c<128 ){
-    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
+    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
  }else if( (unsigned int)c<(1<<22) ){
    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
    int iRes = 0;
@@ -483,7 +472,7 @@ proc print_fold {zFunc} {
  puts "** The results are undefined if the value passed to this function"
  puts "** is less than zero."
  puts "*/"
-  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"
+  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"

  set liOff [tl_generate_ioff_table $lRecord]
  tl_print_table_header
@@ -527,9 +516,7 @@ proc print_fold {zFunc} {
      assert( ret>0 );
    }

-    if( eRemoveDiacritic ){
-      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
-    }
+    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
  }
  }]

@@ -542,259 +529,6 @@ proc print_fold {zFunc} {
  puts "\}"
 }

-proc code {txt} {
-  set txt [string trimright $txt]
-  set txt [string trimleft $txt "\n"]
-  set n [expr {[string length $txt] - [string length [string trim $txt]]}]
-  set ret ""
-  foreach L [split $txt "\n"] {
-    append ret "[string range $L $n end]\n"
-  }
-  return [uplevel "subst -nocommands {$ret}"]
-}
-
-proc intarray {lInt} {
-  set ret ""
-  set n [llength $lInt]
-  for {set i 0} {$i < $n} {incr i 10} {
-    append ret "\n    "
-    foreach int [lrange $lInt $i [expr $i+9]] {
-      append ret [format "%-7s" "$int, "]
-    }
-  }
-  append ret "\n  "
-  set ret
-}
-
-proc categories_switch {Cvar first lSecond} {
-  upvar $Cvar C
-  set ret ""
-  append ret "case '$first':\n"
-  append ret "          switch( zCat\[1\] ){\n"
-  foreach s $lSecond {
-    append ret "            case '$s': aArray\[$C($first$s)\] = 1; break;\n"
-  }
-  append ret "            case '*': \n"
-  foreach s $lSecond {
-    append ret "              aArray\[$C($first$s)\] = 1;\n"
-  }
-  append ret "              break;\n"
-  append ret "            default: return 1;"
-  append ret "          }\n"
-  append ret "          break;\n"
-}
-
-# Argument is a list. Each element of which is itself a list of two elements:
-#
-#   * the codepoint
-#   * the category
-#
-# List elements are sorted in order of codepoint.
-#
-proc print_categories {lMap} {
-  set categories {
-    Cc Cf Cn Cs
-    Ll Lm Lo Lt Lu
-    Mc Me Mn
-    Nd Nl No
-    Pc Pd Pe Pf Pi Po Ps
-    Sc Sk Sm So
-    Zl Zp Zs
-
-    LC Co
-  }
-
-  for {set i 0} {$i < [llength $categories]} {incr i} {
-    set C([lindex $categories $i]) [expr 1+$i]
-  }
-
-  set caseC [categories_switch C C {c f n s o}]
-  set caseL [categories_switch C L {l m o t u C}]
-  set caseM [categories_switch C M {c e n}]
-  set caseN [categories_switch C N {d l o}]
-  set caseP [categories_switch C P {c d e f i o s}]
-  set caseS [categories_switch C S {c k m o}]
-  set caseZ [categories_switch C Z {l p s}]
-
-  set nCat [expr [llength [array names C]] + 1]
-  puts [code {
-    int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
-      aArray[0] = 1;
-      switch( zCat[0] ){
-        $caseC
-        $caseL
-        $caseM
-        $caseN
-        $caseP
-        $caseS
-        $caseZ
-      }
-      return 0;
-    }
-  }]
-
-  set nRepeat 0
-  set first   [lindex $lMap 0 0]
-  set class   [lindex $lMap 0 1]
-  set prev -1
-
-  set CASE(0) "Lu"
-  set CASE(1) "Ll"
-
-  foreach m $lMap {
-    foreach {codepoint cl} $m {}
-    set codepoint [expr "0x$codepoint"]
-    if {$codepoint>=(1<<20)} continue
-
-    set bNew 0
-    if {$codepoint!=($prev+1)} {
-      set bNew 1
-    } elseif {
-      $cl==$class || ($class=="LC" && $cl==$CASE([expr $nRepeat & 0x01]))
-    } {
-      incr nRepeat
-    } elseif {$class=="Lu" && $nRepeat==1 && $cl=="Ll"} {
-      set class LC
-      incr nRepeat
-    } else {
-      set bNew 1
-    }
-    if {$bNew} {
-      lappend lEntries [list $first $class $nRepeat]
-      set nRepeat 1
-      set first $codepoint
-      set class $cl
-    }
-    set prev $codepoint
-  }
-  if {$nRepeat>0} {
-    lappend lEntries [list $first $class $nRepeat]
-  }
-
-  set aBlock [list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
-  set aMap [list]
-  foreach e $lEntries {
-    foreach {cp class nRepeat} $e {}
-    set block [expr ($cp>>16)]
-    if {$block>0 && [lindex $aBlock $block]==0} {
-      for {set i 1} {$i<=$block} {incr i} {
-        if {[lindex $aBlock $i]==0} {
-          lset aBlock $i [llength $aMap]
-        }
-      }
-    }
-    lappend aMap [expr {$cp & 0xFFFF}]
-    lappend aData [expr {($nRepeat << 5) + $C($class)}]
-  }
-  for {set i 1} {$i<[llength $aBlock]} {incr i} {
-    if {[lindex $aBlock $i]==0} {
-      lset aBlock $i [llength $aMap]
-    }
-  }
-
-  set aBlockArray [intarray $aBlock]
-  set aMapArray [intarray $aMap]
-  set aDataArray [intarray $aData]
-  puts [code {
-    static u16 aFts5UnicodeBlock[] = {$aBlockArray};
-    static u16 aFts5UnicodeMap[] = {$aMapArray};
-    static u16 aFts5UnicodeData[] = {$aDataArray};
-
-    int sqlite3Fts5UnicodeCategory(u32 iCode) { 
-      int iRes = -1;
-      int iHi;
-      int iLo;
-      int ret;
-      u16 iKey;
-
-      if( iCode>=(1<<20) ){
-        return 0;
-      }
-      iLo = aFts5UnicodeBlock[(iCode>>16)];
-      iHi = aFts5UnicodeBlock[1+(iCode>>16)];
-      iKey = (iCode & 0xFFFF);
-      while( iHi>iLo ){
-        int iTest = (iHi + iLo) / 2;
-        assert( iTest>=iLo && iTest<iHi );
-        if( iKey>=aFts5UnicodeMap[iTest] ){
-          iRes = iTest;
-          iLo = iTest+1;
-        }else{
-          iHi = iTest;
-        }
-      }
-
-      if( iRes<0 ) return 0;
-      if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
-      ret = aFts5UnicodeData[iRes] & 0x1F;
-      if( ret!=$C(LC) ) return ret;
-      return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? $C(Ll) : $C(Lu);
-    }
-
-    void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
-      int i = 0;
-      int iTbl = 0;
-      while( i<128 ){
-        int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
-        int n = (aFts5UnicodeData[iTbl] >> 5) + i;
-        for(; i<128 && i<n; i++){
-          aAscii[i] = (u8)bToken;
-        }
-        iTbl++;
-      }
-    }
-  }]
-}
-
-proc print_test_categories {lMap} {
-
-  set lCP [list]
-  foreach e $lMap {
-    foreach {cp cat} $e {}
-    if {[expr 0x$cp] < (1<<20)} {
-      lappend lCP "{0x$cp, \"$cat\"}, "
-    }
-  }
-
-  set aCP "\n"
-  for {set i 0} {$i < [llength $lCP]} {incr i 4} {
-    append aCP "    [join [lrange $lCP $i $i+3]]\n"
-  }
-
-
-  puts [code {
-    static int categories_test (int *piCode){
-      struct Codepoint {
-        int iCode;
-        const char *zCat;
-      } aCP[] = {$aCP};
-      int i;
-      int iCP = 0;
-
-      for(i=0; i<1000000; i++){
-        u8 aArray[40];
-        int cat = 0;
-        int c = 0;
-        memset(aArray, 0, sizeof(aArray));
-        if( aCP[iCP].iCode==i ){
-          sqlite3Fts5UnicodeCatParse(aCP[iCP].zCat, aArray);
-          iCP++;
-        }else{
-          aArray[0] = 1;
-        }
-
-        c = sqlite3Fts5UnicodeCategory((u32)i);
-        if( aArray[c]==0 ){
-          *piCode = i;
-          return 1;
-        }
-      }
-
-      return 0;
-    }
-  }]
-}
-
 proc print_fold_test {zFunc mappings} {
  global tl_lookup_table

@@ -838,7 +572,7 @@ proc print_fold_test {zFunc mappings} {
 proc print_fileheader {} {
  puts [string trim {
 /*
-** 2012-05-25
+** 2012 May 25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -871,21 +605,15 @@ proc print_test_main {} {
  puts "#include <stdio.h>"
  puts ""
  puts "int main(int argc, char **argv)\{"
-  puts "  int r1, r2, r3;"
+  puts "  int r1, r2;"
  puts "  int code;"
-  puts "  r3 = 0;"
  puts "  r1 = isalnum_test(&code);"
  puts "  if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);"
  puts "  else printf(\"isalnum(): test passed\\n\");"
  puts "  r2 = fold_test(&code);"
  puts "  if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);"
  puts "  else printf(\"fold(): test passed\\n\");"
-  if {$::generate_fts5_code} {
-    puts "  r3 = categories_test(&code);"
-    puts "  if( r3 ) printf(\"categories(): Problem with code %d\\n\",code);"
-    puts "  else printf(\"categories(): test passed\\n\");"
-  }
-  puts "  return (r1 || r2 || r3);"
+  puts "  return (r1 || r2);"
  puts "\}"
 }

@@ -923,18 +651,10 @@ for {set i 0} {$i < [llength $argv]-2} {incr i} {

 print_fileheader

-if {$::generate_test_code} {
-  puts "typedef unsigned short int u16;"
-  puts "typedef unsigned char u8;"
-  puts "#include <string.h>"
-}
-
 # Print the isalnum() function to stdout.
 #
 set lRange [an_load_separator_ranges]
-if {$generate_fts5_code==0} {
-  print_isalnum ${function_prefix}UnicodeIsalnum $lRange
-}
+print_isalnum ${function_prefix}UnicodeIsalnum $lRange

 # Leave a gap between the two generated C functions.
 #
@@ -957,21 +677,12 @@ puts ""
 #
 print_fold ${function_prefix}UnicodeFold

-if {$generate_fts5_code} {
-  puts ""
-  puts ""
-  print_categories [cc_load_unicodedata_text ${unicodedata.txt}]
-}
-
 # Print the test routines and main() function to stdout, if -test 
 # was specified.
 #
 if {$::generate_test_code} {
-  if {$generate_fts5_code==0} {
-    print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
-  }
+  print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
  print_fold_test ${function_prefix}UnicodeFold $mappings
-  print_test_categories [cc_load_unicodedata_text ${unicodedata.txt}]
  print_test_main 
 }

@@ -7,24 +7,12 @@
 # character that it should be replaced with, or an empty string if the 
 # codepoint should simply be removed from the input. Examples:
 #
-#   { 224 a  0 }     (replace codepoint 224 to "a")
-#   { 769 "" 0 }     (remove codepoint 769 from input)
+#   { 224 a  }     (replace codepoint 224 to "a")
+#   { 769 "" }     (remove codepoint 769 from input)
 #
 # Mappings are only returned for non-upper case codepoints. It is assumed
 # that the input has already been folded to lower case.
 #
-# The third value in the list is always either 0 or 1. 0 if the 
-# UnicodeData.txt file maps the codepoint to a single ASCII character and
-# a diacritic, or 1 if the mapping is indirect. For example, consider the 
-# two entries:
-#
-# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
-# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
-#
-# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
-# diacritic). The second is an indirect mapping, as it maps to the
-# first codepoint plus 0302 (a diacritic).
-#
 proc rd_load_unicodedata_text {zName} {
  global tl_lookup_table

@@ -65,29 +53,18 @@ proc rd_load_unicodedata_text {zName} {
    set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
    set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]

-    # Filter out upper-case characters, as they will be mapped to their
-    # lower-case equivalents before this data is used.
    if {[info exists tl_lookup_table($iCode)]} continue

-    # Check if this is an indirect mapping. If so, set bIndirect to true
-    # and change $iAscii to the indirectly mappped ASCII character.
-    set bIndirect 0
-    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
-      set iAscii $mapping($iAscii)
-      set bIndirect 1
-    }
-
    if { ($iAscii >= 97 && $iAscii <= 122)
      || ($iAscii >= 65 && $iAscii <= 90)
    } {
-      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
-      set mapping($iCode) $iAscii
+      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
      set dia($iDia) 1
    }
  }

  foreach d [array names dia] {
-    lappend lRet [list $d "" 0]
+    lappend lRet [list $d ""]
  }
  set lRet [lsort -integer -index 0 $lRet]

@@ -166,40 +143,4 @@ proc tl_load_casefolding_txt {zName} {
  }
 }

-proc cc_load_unicodedata_text {zName} {
-  set fd [open $zName]
-  set lField {
-    code
-    character_name
-    general_category
-    canonical_combining_classes
-    bidirectional_category
-    character_decomposition_mapping
-    decimal_digit_value
-    digit_value
-    numeric_value
-    mirrored
-    unicode_1_name
-    iso10646_comment_field
-    uppercase_mapping
-    lowercase_mapping
-    titlecase_mapping
-  }
-  set lRet [list]
-
-  while { ![eof $fd] } {
-    set line [gets $fd]
-    if {$line == ""} continue
-
-    set fields [split $line ";"]
-    if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
-    foreach $lField $fields {}
-
-    lappend lRet [list $code $general_category]
-  }
-
-  close $fd
-  set lRet
-}
-

@@ -120,8 +120,12 @@ struct Fts5PhraseIter {
 **
 **   Usually, output parameter *piPhrase is set to the phrase number, *piCol
 **   to the column in which it occurs and *piOff the token offset of the
-**   first token of the phrase. Returns SQLITE_OK if successful, or an error
-**   code (i.e. SQLITE_NOMEM) if an error occurs.
+**   first token of the phrase. The exception is if the table was created
+**   with the offsets=0 option specified. In this case *piOff is always
+**   set to -1.
+**
+**   Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) 
+**   if an error occurs.
 **
 **   This API can be quite slow if used with an FTS5 table created with the
 **   "detail=none" or "detail=column" option. 
@@ -159,10 +163,10 @@ struct Fts5PhraseIter {
 **
 ** xSetAuxdata(pFts5, pAux, xDelete)
 **
-**   Save the pointer passed as the second argument as the extension function's 
+**   Save the pointer passed as the second argument as the extension functions 
 **   "auxiliary data". The pointer may then be retrieved by the current or any
 **   future invocation of the same fts5 extension function made as part of
-**   the same MATCH query using the xGetAuxdata() API.
+**   of the same MATCH query using the xGetAuxdata() API.
 **
 **   Each extension function is allocated a single auxiliary data slot for
 **   each FTS query (MATCH expression). If the extension function is invoked 
@@ -177,7 +181,7 @@ struct Fts5PhraseIter {
 **   The xDelete callback, if one is specified, is also invoked on the
 **   auxiliary data pointer after the FTS5 query has finished.
 **
-**   If an error (e.g. an OOM condition) occurs within this function,
+**   If an error (e.g. an OOM condition) occurs within this function, an
 **   the auxiliary data is set to NULL and an error code returned. If the
 **   xDelete parameter was not NULL, it is invoked on the auxiliary data
 **   pointer before returning.
@@ -401,8 +405,8 @@ struct Fts5ExtensionApi {
 **
 **   There are several ways to approach this in FTS5:
 **
-**   <ol><li> By mapping all synonyms to a single token. In this case, using
-**            the above example, this means that the tokenizer returns the
+**   <ol><li> By mapping all synonyms to a single token. In this case, the 
+**            In the above example, this means that the tokenizer returns the
 **            same token for inputs "first" and "1st". Say that token is in
 **            fact "first", so that when the user inserts the document "I won
 **            1st place" entries are added to the index for tokens "i", "won",
@@ -410,11 +414,11 @@ struct Fts5ExtensionApi {
 **            the tokenizer substitutes "first" for "1st" and the query works
 **            as expected.
 **
-**       <li> By querying the index for all synonyms of each query term
-**            separately. In this case, when tokenizing query text, the
-**            tokenizer may provide multiple synonyms for a single term 
-**            within the document. FTS5 then queries the index for each 
-**            synonym individually. For example, faced with the query:
+**       <li> By adding multiple synonyms for a single term to the FTS index.
+**            In this case, when tokenizing query text, the tokenizer may 
+**            provide multiple synonyms for a single term within the document.
+**            FTS5 then queries the index for each synonym individually. For
+**            example, faced with the query:
 **
 **   <codeblock>
 **     ... MATCH 'first place'</codeblock>
@@ -438,9 +442,9 @@ struct Fts5ExtensionApi {
 **            "place".
 **
 **            This way, even if the tokenizer does not provide synonyms
-**            when tokenizing query text (it should not - to do so would be
+**            when tokenizing query text (it should not - to do would be
 **            inefficient), it doesn't matter if the user queries for 
-**            'first + place' or '1st + place', as there are entries in the
+**            'first + place' or '1st + place', as there are entires in the
 **            FTS index corresponding to both forms of the first token.
 **   </ol>
 **
@@ -468,7 +472,7 @@ struct Fts5ExtensionApi {
 **   extra data to the FTS index or require FTS5 to query for multiple terms,
 **   so it is efficient in terms of disk space and query speed. However, it
 **   does not support prefix queries very well. If, as suggested above, the
-**   token "first" is substituted for "1st" by the tokenizer, then the query:
+**   token "first" is subsituted for "1st" by the tokenizer, then the query:
 **
 **   <codeblock>
 **     ... MATCH '1s*'</codeblock>
@@ -30,9 +30,7 @@ typedef short i16;
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;

-#ifndef ArraySize
-# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
-#endif
+#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))

 #define testcase(x)
 #define ALWAYS(x) 1
@@ -61,11 +59,6 @@ typedef sqlite3_uint64 u64;
 */
 #define FTS5_MAX_PREFIX_INDEXES 31

-/*
-** Maximum segments permitted in a single index 
-*/
-#define FTS5_MAX_SEGMENT 2000
-
 #define FTS5_DEFAULT_NEARDIST 10
 #define FTS5_DEFAULT_RANK     "bm25"

@@ -92,12 +85,6 @@ extern int sqlite3_fts5_may_be_corrupt;
 # define assert_nc(x) assert(x)
 #endif

-/*
-** A version of memcmp() that does not cause asan errors if one of the pointer
-** parameters is NULL and the number of bytes to compare is zero.
-*/
-#define fts5Memcmp(s1, s2, n) ((n)==0 ? 0 : memcmp((s1), (s2), (n)))
-
 /* Mark a function parameter as unused, to suppress nuisance compiler
 ** warnings. */
 #ifndef UNUSED_PARAM
@@ -183,7 +170,6 @@ struct Fts5Config {
  char *zContentExprlist;
  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;
-  int bLock;                      /* True when table is preparing statement */

  /* Values loaded from the %_config table */
  int iCookie;                    /* Incremented when %_config is modified */
@@ -286,7 +272,7 @@ void sqlite3Fts5Put32(u8*, int);
 int sqlite3Fts5Get32(const u8*);

 #define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
-#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)
+#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)

 typedef struct Fts5PoslistReader Fts5PoslistReader;
 struct Fts5PoslistReader {
@@ -321,7 +307,7 @@ int sqlite3Fts5PoslistNext64(
 );

 /* Malloc utility */
-void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte);
+void *sqlite3Fts5MallocZero(int *pRc, int nByte);
 char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);

 /* Character set tests (like isspace(), isalpha() etc.) */
@@ -422,11 +408,6 @@ int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
 */
 void sqlite3Fts5IterClose(Fts5IndexIter*);

-/*
-** Close the reader blob handle, if it is open.
-*/
-void sqlite3Fts5IndexCloseReader(Fts5Index*);
-
 /*
 ** This interface is used by the fts5vocab module.
 */
@@ -463,9 +444,9 @@ int sqlite3Fts5IndexBeginWrite(

 /*
 ** Flush any data stored in the in-memory hash tables to the database.
-** Also close any open blob handles.
+** If the bCommit flag is true, also close any open blob handles.
 */
-int sqlite3Fts5IndexSync(Fts5Index *p);
+int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit);

 /*
 ** Discard any data stored in the in-memory hash tables. Do not write it
@@ -537,19 +518,9 @@ int sqlite3Fts5PutVarint(unsigned char *p, u64 v);


 /**************************************************************************
-** Interface to code in fts5_main.c. 
+** Interface to code in fts5.c. 
 */

-/*
-** Virtual-table object.
-*/
-typedef struct Fts5Table Fts5Table;
-struct Fts5Table {
-  sqlite3_vtab base;              /* Base class used by SQLite core */
-  Fts5Config *pConfig;            /* Virtual table configuration */
-  Fts5Index *pIndex;              /* Full-text index */
-};
-
 int sqlite3Fts5GetTokenizer(
  Fts5Global*, 
  const char **azArg,
@@ -559,9 +530,7 @@ int sqlite3Fts5GetTokenizer(
  char **pzErr
 );

-Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
-
-int sqlite3Fts5FlushToDisk(Fts5Table*);
+Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **);

 /*
 ** End of interface to code in fts5.c.
@@ -594,9 +563,8 @@ void sqlite3Fts5HashClear(Fts5Hash*);

 int sqlite3Fts5HashQuery(
  Fts5Hash*,                      /* Hash table to query */
-  int nPre,
  const char *pTerm, int nTerm,   /* Query term */
-  void **ppObj,                   /* OUT: Pointer to doclist for pTerm */
+  const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
  int *pnDoclist                  /* OUT: Size of doclist in bytes */
 );

@@ -648,7 +616,7 @@ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);

-int sqlite3Fts5StorageSync(Fts5Storage *p);
+int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
 int sqlite3Fts5StorageRollback(Fts5Storage *p);

 int sqlite3Fts5StorageConfigValue(
@@ -684,7 +652,6 @@ struct Fts5Token {
 /* Parse a MATCH expression. */
 int sqlite3Fts5ExprNew(
  Fts5Config *pConfig, 
-  int iCol,                       /* Column on LHS of MATCH operator */
  const char *zExpr,
  Fts5Expr **ppNew, 
  char **pzErr
@@ -705,7 +672,6 @@ int sqlite3Fts5ExprEof(Fts5Expr*);
 i64 sqlite3Fts5ExprRowid(Fts5Expr*);

 void sqlite3Fts5ExprFree(Fts5Expr*);
-int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2);

 /* Called during startup to register a UDF with SQLite */
 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
@@ -753,8 +719,6 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
  int bPrefix
 );

-void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
-
 Fts5ExprNearset *sqlite3Fts5ParseNearset(
  Fts5Parse*, 
  Fts5ExprNearset*,
@@ -772,7 +736,7 @@ void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
-void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
+void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
 Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
@@ -815,12 +779,9 @@ int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
 /**************************************************************************
 ** Interface to automatically generated code in fts5_unicode2.c. 
 */
+int sqlite3Fts5UnicodeIsalnum(int c);
 int sqlite3Fts5UnicodeIsdiacritic(int c);
 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
-
-int sqlite3Fts5UnicodeCatParse(const char*, u8*);
-int sqlite3Fts5UnicodeCategory(u32 iCode);
-void sqlite3Fts5UnicodeAscii(u8*, u8*);
 /*
 ** End of interface to code in fts5_unicode2.c.
 **************************************************************************/
@@ -136,7 +136,7 @@ static void fts5HighlightAppend(
  HighlightContext *p, 
  const char *z, int n
 ){
-  if( *pRc==SQLITE_OK && z ){
+  if( *pRc==SQLITE_OK ){
    if( n<0 ) n = (int)strlen(z);
    p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
    if( p->zOut==0 ) *pRc = SQLITE_NOMEM;
@@ -268,7 +268,7 @@ static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
    int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
    int *aNew;

-    aNew = (int*)sqlite3_realloc64(p->aFirst, nNew*sizeof(int));
+    aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int));
    if( aNew==0 ) return SQLITE_NOMEM;
    p->aFirst = aNew;
    p->nFirstAlloc = nNew;
@@ -335,12 +335,11 @@ static int fts5SnippetScore(
  int nInst;
  int nScore = 0;
  int iLast = 0;
-  sqlite3_int64 iEnd = (sqlite3_int64)iPos + nToken;

  rc = pApi->xInstCount(pFts, &nInst);
  for(i=0; i<nInst && rc==SQLITE_OK; i++){
    rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
-    if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<iEnd ){
+    if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){
      nScore += (aSeen[ip] ? 1 : 1000);
      aSeen[ip] = 1;
      if( iFirst<0 ) iFirst = iOff;
@@ -350,25 +349,15 @@ static int fts5SnippetScore(

  *pnScore = nScore;
  if( piPos ){
-    sqlite3_int64 iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
+    int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
    if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
    if( iAdj<0 ) iAdj = 0;
-    *piPos = (int)iAdj;
+    *piPos = iAdj;
  }

  return rc;
 }

-/*
-** Return the value in pVal interpreted as utf-8 text. Except, if pVal 
-** contains a NULL value, return a pointer to a static string zero
-** bytes in length instead of a NULL pointer.
-*/
-static const char *fts5ValueToText(sqlite3_value *pVal){
-  const char *zRet = (const char*)sqlite3_value_text(pVal);
-  return zRet ? zRet : "";
-}
-
 /*
 ** Implementation of snippet() function.
 */
@@ -404,9 +393,9 @@ static void fts5SnippetFunction(
  nCol = pApi->xColumnCount(pFts);
  memset(&ctx, 0, sizeof(HighlightContext));
  iCol = sqlite3_value_int(apVal[0]);
-  ctx.zOpen = fts5ValueToText(apVal[1]);
-  ctx.zClose = fts5ValueToText(apVal[2]);
-  zEllips = fts5ValueToText(apVal[3]);
+  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
+  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
+  zEllips = (const char*)sqlite3_value_text(apVal[3]);
  nToken = sqlite3_value_int(apVal[4]);

  iBestCol = (iCol>=0 ? iCol : 0);
@@ -443,9 +432,7 @@ static void fts5SnippetFunction(
        int jj;

        rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
-        if( ic!=i ) continue;
-        if( io>nDocsize ) rc = FTS5_CORRUPT;
-        if( rc!=SQLITE_OK ) continue;
+        if( ic!=i || rc!=SQLITE_OK ) continue;
        memset(aSeen, 0, nPhrase);
        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
            io, nToken, &nScore, &iAdj
@@ -571,17 +558,17 @@ static int fts5Bm25GetData(
    int nPhrase;                  /* Number of phrases in query */
    sqlite3_int64 nRow = 0;       /* Number of rows in table */
    sqlite3_int64 nToken = 0;     /* Number of tokens in table */
-    sqlite3_int64 nByte;          /* Bytes of space to allocate */
+    int nByte;                    /* Bytes of space to allocate */
    int i;

    /* Allocate the Fts5Bm25Data object */
    nPhrase = pApi->xPhraseCount(pFts);
    nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double);
-    p = (Fts5Bm25Data*)sqlite3_malloc64(nByte);
+    p = (Fts5Bm25Data*)sqlite3_malloc(nByte);
    if( p==0 ){
      rc = SQLITE_NOMEM;
    }else{
-      memset(p, 0, (size_t)nByte);
+      memset(p, 0, nByte);
      p->nPhrase = nPhrase;
      p->aIDF = (double*)&p[1];
      p->aFreq = &p->aIDF[nPhrase];
@@ -589,7 +576,6 @@ static int fts5Bm25GetData(

    /* Calculate the average document length for this FTS5 table */
    if( rc==SQLITE_OK ) rc = pApi->xRowCount(pFts, &nRow);
-    assert( rc!=SQLITE_OK || nRow>0 );
    if( rc==SQLITE_OK ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken);
    if( rc==SQLITE_OK ) p->avgdl = (double)nToken  / (double)nRow;

@@ -714,3 +700,5 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){

  return rc;
 }
+
+
@@ -17,17 +17,17 @@

 int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
  if( (u32)pBuf->nSpace<nByte ){
-    u64 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
+    u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
    u8 *pNew;
    while( nNew<nByte ){
      nNew = nNew * 2;
    }
-    pNew = sqlite3_realloc64(pBuf->p, nNew);
+    pNew = sqlite3_realloc(pBuf->p, nNew);
    if( pNew==0 ){
      *pRc = SQLITE_NOMEM;
      return 1;
    }else{
-      pBuf->nSpace = (int)nNew;
+      pBuf->nSpace = nNew;
      pBuf->p = pNew;
    }
  }
@@ -52,7 +52,7 @@ void sqlite3Fts5Put32(u8 *aBuf, int iVal){
 }

 int sqlite3Fts5Get32(const u8 *aBuf){
-  return (int)((((u32)aBuf[0])<<24) + (aBuf[1]<<16) + (aBuf[2]<<8) + aBuf[3]);
+  return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3];
 }

 /*
@@ -67,11 +67,9 @@ void sqlite3Fts5BufferAppendBlob(
  const u8 *pData
 ){
  assert_nc( *pRc || nData>=0 );
-  if( nData ){
-    if( fts5BufferGrow(pRc, pBuf, nData) ) return;
-    memcpy(&pBuf->p[pBuf->n], pData, nData);
-    pBuf->n += nData;
-  }
+  if( fts5BufferGrow(pRc, pBuf, nData) ) return;
+  memcpy(&pBuf->p[pBuf->n], pData, nData);
+  pBuf->n += nData;
 }

 /*
@@ -178,21 +176,12 @@ int sqlite3Fts5PoslistNext64(
    i64 iOff = *piOff;
    int iVal;
    fts5FastGetVarint32(a, i, iVal);
-    if( iVal<=1 ){
-      if( iVal==0 ){
-        *pi = i;
-        return 0;
-      }
+    if( iVal==1 ){
      fts5FastGetVarint32(a, i, iVal);
      iOff = ((i64)iVal) << 32;
      fts5FastGetVarint32(a, i, iVal);
-      if( iVal<2 ){
-        /* This is a corrupt record. So stop parsing it here. */
-        *piOff = -1;
-        return 1;
-      }
    }
-    *piOff = iOff + ((iVal-2) & 0x7FFFFFFF);
+    *piOff = iOff + (iVal-2);
    *pi = i;
    return 0;
  }
@@ -253,14 +242,14 @@ int sqlite3Fts5PoslistWriterAppend(
  return SQLITE_OK;
 }

-void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte){
+void *sqlite3Fts5MallocZero(int *pRc, int nByte){
  void *pRet = 0;
  if( *pRc==SQLITE_OK ){
-    pRet = sqlite3_malloc64(nByte);
-    if( pRet==0 ){
-      if( nByte>0 ) *pRc = SQLITE_NOMEM;
+    pRet = sqlite3_malloc(nByte);
+    if( pRet==0 && nByte>0 ){
+      *pRc = SQLITE_NOMEM;
    }else{
-      memset(pRet, 0, (size_t)nByte);
+      memset(pRet, 0, nByte);
    }
  }
  return pRet;
@@ -23,7 +23,7 @@
 #define FTS5_DEFAULT_HASHSIZE    (1024*1024)

 /* Maximum allowed page size */
-#define FTS5_MAX_PAGE_SIZE (64*1024)
+#define FTS5_MAX_PAGE_SIZE (128*1024)

 static int fts5_iswhitespace(char x){
  return (x==' ');
@@ -150,7 +150,7 @@ static int fts5Dequote(char *z){
  assert( q=='[' || q=='\'' || q=='"' || q=='`' );
  if( q=='[' ) q = ']';  

-  while( z[iIn] ){
+  while( ALWAYS(z[iIn]) ){
    if( z[iIn]==q ){
      if( z[iIn+1]!=q ){
        /* Character iIn was the close quote. */
@@ -295,7 +295,7 @@ static int fts5ConfigParseSpecial(

  if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
    const char *p = (const char*)zArg;
-    sqlite3_int64 nArg = strlen(zArg) + 1;
+    int nArg = (int)strlen(zArg) + 1;
    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
    char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
    char *pSpace = pDel;
@@ -325,7 +325,7 @@ static int fts5ConfigParseSpecial(
          rc = SQLITE_ERROR;
        }else{
          rc = sqlite3Fts5GetTokenizer(pGlobal, 
-              (const char**)azArg, (int)nArg, &pConfig->pTok, &pConfig->pTokApi,
+              (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi,
              pzErr
          );
        }
@@ -425,8 +425,8 @@ static const char *fts5ConfigGobbleWord(
 ){
  const char *zRet = 0;

-  sqlite3_int64 nIn = strlen(zIn);
-  char *zOut = sqlite3_malloc64(nIn+1);
+  int nIn = (int)strlen(zIn);
+  char *zOut = sqlite3_malloc(nIn+1);

  assert( *pRc==SQLITE_OK );
  *pbQuoted = 0;
@@ -435,7 +435,7 @@ static const char *fts5ConfigGobbleWord(
  if( zOut==0 ){
    *pRc = SQLITE_NOMEM;
  }else{
-    memcpy(zOut, zIn, (size_t)(nIn+1));
+    memcpy(zOut, zIn, nIn+1);
    if( fts5_isopenquote(zOut[0]) ){
      int ii = fts5Dequote(zOut);
      zRet = &zIn[ii];
@@ -529,7 +529,7 @@ int sqlite3Fts5ConfigParse(
  int rc = SQLITE_OK;             /* Return code */
  Fts5Config *pRet;               /* New object to return */
  int i;
-  sqlite3_int64 nByte;
+  int nByte;

  *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
  if( pRet==0 ) return SQLITE_NOMEM;
@@ -683,7 +683,7 @@ int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
    rc = sqlite3_declare_vtab(pConfig->db, zSql);
    sqlite3_free(zSql);
  }
- 
+  
  return rc;
 }

@@ -828,7 +828,7 @@ int sqlite3Fts5ConfigSetValue(
    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
      pgsz = sqlite3_value_int(pVal);
    }
-    if( pgsz<32 || pgsz>FTS5_MAX_PAGE_SIZE ){
+    if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){
      *pbBadkey = 1;
    }else{
      pConfig->pgsz = pgsz;
@@ -881,7 +881,6 @@ int sqlite3Fts5ConfigSetValue(
      *pbBadkey = 1;
    }else{
      if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
-      if( nCrisisMerge>=FTS5_MAX_SEGMENT ) nCrisisMerge = FTS5_MAX_SEGMENT-1;
      pConfig->nCrisisMerge = nCrisisMerge;
    }
  }
@@ -36,7 +36,6 @@ void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
 #include <stdio.h>
 void sqlite3Fts5ParserTrace(FILE*, char*);
 #endif
-int sqlite3Fts5ParserFallback(int);


 struct Fts5Expr {
@@ -88,8 +87,7 @@ struct Fts5ExprNode {
 ** or term prefix.
 */
 struct Fts5ExprTerm {
-  u8 bPrefix;                     /* True for a prefix term */
-  u8 bFirst;                      /* True if token must be first in column */
+  int bPrefix;                    /* True for a prefix term */
  char *zTerm;                    /* nul-terminated term */
  Fts5IndexIter *pIter;           /* Iterator for this term */
  Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
@@ -170,7 +168,6 @@ static int fts5ExprGetToken(
    case '+':  tok = FTS5_PLUS;  break;
    case '*':  tok = FTS5_STAR;  break;
    case '-':  tok = FTS5_MINUS; break;
-    case '^':  tok = FTS5_CARET; break;
    case '\0': tok = FTS5_EOF;   break;

    case '"': {
@@ -211,12 +208,11 @@ static int fts5ExprGetToken(
  return tok;
 }

-static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc64((sqlite3_int64)t);}
+static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc((int)t); }
 static void fts5ParseFree(void *p){ sqlite3_free(p); }

 int sqlite3Fts5ExprNew(
  Fts5Config *pConfig,            /* FTS5 Configuration */
-  int iCol,
  const char *zExpr,              /* Expression text */
  Fts5Expr **ppNew, 
  char **pzErr
@@ -241,18 +237,6 @@ int sqlite3Fts5ExprNew(
  }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
  sqlite3Fts5ParserFree(pEngine, fts5ParseFree);

-  /* If the LHS of the MATCH expression was a user column, apply the
-  ** implicit column-filter.  */
-  if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
-    int n = sizeof(Fts5Colset);
-    Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
-    if( pColset ){
-      pColset->nCol = 1;
-      pColset->aiCol[0] = iCol;
-      sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset);
-    }
-  }
-
  assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
  if( sParse.rc==SQLITE_OK ){
    *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
@@ -309,42 +293,6 @@ void sqlite3Fts5ExprFree(Fts5Expr *p){
  }
 }

-int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
-  Fts5Parse sParse;
-  memset(&sParse, 0, sizeof(sParse));
-
-  if( *pp1 ){
-    Fts5Expr *p1 = *pp1;
-    int nPhrase = p1->nPhrase + p2->nPhrase;
-
-    p1->pRoot = sqlite3Fts5ParseNode(&sParse, FTS5_AND, p1->pRoot, p2->pRoot,0);
-    p2->pRoot = 0;
-
-    if( sParse.rc==SQLITE_OK ){
-      Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc(
-          p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*)
-      );
-      if( ap==0 ){
-        sParse.rc = SQLITE_NOMEM;
-      }else{
-        int i;
-        memmove(&ap[p2->nPhrase], ap, p1->nPhrase*sizeof(Fts5ExprPhrase*));
-        for(i=0; i<p2->nPhrase; i++){
-          ap[i] = p2->apExprPhrase[i];
-        }
-        p1->nPhrase = nPhrase;
-        p1->apExprPhrase = ap;
-      }
-    }
-    sqlite3_free(p2->apExprPhrase);
-    sqlite3_free(p2);
-  }else{
-    *pp1 = p2;
-  }
-
-  return sParse.rc;
-}
-
 /*
 ** Argument pTerm must be a synonym iterator. Return the current rowid
 ** that it points to.
@@ -392,8 +340,8 @@ static int fts5ExprSynonymList(
    if( sqlite3Fts5IterEof(pIter)==0 && pIter->iRowid==iRowid ){
      if( pIter->nData==0 ) continue;
      if( nIter==nAlloc ){
-        sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
-        Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc64(nByte);
+        int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
+        Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
        if( aNew==0 ){
          rc = SQLITE_NOMEM;
          goto synonym_poslist_out;
@@ -466,15 +414,14 @@ static int fts5ExprPhraseIsMatch(
  Fts5PoslistReader *aIter = aStatic;
  int i;
  int rc = SQLITE_OK;
-  int bFirst = pPhrase->aTerm[0].bFirst;
  
  fts5BufferZero(&pPhrase->poslist);

  /* If the aStatic[] array is not large enough, allocate a large array
  ** using sqlite3_malloc(). This approach could be improved upon. */
  if( pPhrase->nTerm>ArraySize(aStatic) ){
-    sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
-    aIter = (Fts5PoslistReader*)sqlite3_malloc64(nByte);
+    int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
+    aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte);
    if( !aIter ) return SQLITE_NOMEM;
  }
  memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm);
@@ -521,10 +468,8 @@ static int fts5ExprPhraseIsMatch(
    }while( bMatch==0 );

    /* Append position iPos to the output */
-    if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){
-      rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
-      if( rc!=SQLITE_OK ) goto ismatch_out;
-    }
+    rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
+    if( rc!=SQLITE_OK ) goto ismatch_out;

    for(i=0; i<pPhrase->nTerm; i++){
      if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
@@ -608,7 +553,7 @@ static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){
  /* If the aStatic[] array is not large enough, allocate a large array
  ** using sqlite3_malloc(). This approach could be improved upon. */
  if( pNear->nPhrase>ArraySize(aStatic) ){
-    sqlite3_int64 nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
+    int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
    a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte);
  }else{
    memset(aStatic, 0, sizeof(aStatic));
@@ -778,9 +723,7 @@ static int fts5ExprNearTest(
    ** phrase is not a match, break out of the loop early.  */
    for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
-      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym 
-       || pNear->pColset || pPhrase->aTerm[0].bFirst
-      ){
+      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
        int bMatch = 0;
        rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
        if( bMatch==0 ) break;
@@ -803,61 +746,48 @@ static int fts5ExprNearTest(
 ** Initialize all term iterators in the pNear object. If any term is found
 ** to match no documents at all, return immediately without initializing any
 ** further iterators.
-**
-** If an error occurs, return an SQLite error code. Otherwise, return
-** SQLITE_OK. It is not considered an error if some term matches zero
-** documents.
 */
 static int fts5ExprNearInitAll(
  Fts5Expr *pExpr,
  Fts5ExprNode *pNode
 ){
  Fts5ExprNearset *pNear = pNode->pNear;
-  int i;
+  int i, j;
+  int rc = SQLITE_OK;
+  int bEof = 1;

  assert( pNode->bNomatch==0 );
-  for(i=0; i<pNear->nPhrase; i++){
+  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
    Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
-    if( pPhrase->nTerm==0 ){
-      pNode->bEof = 1;
-      return SQLITE_OK;
-    }else{
-      int j;
-      for(j=0; j<pPhrase->nTerm; j++){
-        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
-        Fts5ExprTerm *p;
-        int bHit = 0;
+    for(j=0; j<pPhrase->nTerm; j++){
+      Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
+      Fts5ExprTerm *p;

-        for(p=pTerm; p; p=p->pSynonym){
-          int rc;
-          if( p->pIter ){
-            sqlite3Fts5IterClose(p->pIter);
-            p->pIter = 0;
-          }
-          rc = sqlite3Fts5IndexQuery(
-              pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
-              (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
-              (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
-              pNear->pColset,
-              &p->pIter
-          );
-          assert( (rc==SQLITE_OK)==(p->pIter!=0) );
-          if( rc!=SQLITE_OK ) return rc;
-          if( 0==sqlite3Fts5IterEof(p->pIter) ){
-            bHit = 1;
-          }
+      for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){
+        if( p->pIter ){
+          sqlite3Fts5IterClose(p->pIter);
+          p->pIter = 0;
        }
-
-        if( bHit==0 ){
-          pNode->bEof = 1;
-          return SQLITE_OK;
+        rc = sqlite3Fts5IndexQuery(
+            pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
+            (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
+            (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
+            pNear->pColset,
+            &p->pIter
+        );
+        assert( rc==SQLITE_OK || p->pIter==0 );
+        if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
+          bEof = 0;
        }
      }
+
+      if( bEof ) break;
    }
+    if( bEof ) break;
  }

-  pNode->bEof = 0;
-  return SQLITE_OK;
+  pNode->bEof = bEof;
+  return rc;
 }

 /*
@@ -961,7 +891,6 @@ static int fts5ExprNodeTest_STRING(
  assert( pNear->nPhrase>1 
       || pNear->apPhrase[0]->nTerm>1 
       || pNear->apPhrase[0]->aTerm[0].pSynonym
-       || pNear->apPhrase[0]->aTerm[0].bFirst
  );

  /* Initialize iLast, the "lastest" rowid any iterator points to. If the
@@ -991,7 +920,7 @@ static int fts5ExprNodeTest_STRING(
          }
        }else{
          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
-          if( pIter->iRowid==iLast || pIter->bEof ) continue;
+          if( pIter->iRowid==iLast ) continue;
          bMatch = 0;
          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
            return rc;
@@ -1168,10 +1097,7 @@ static int fts5ExprNodeNext_OR(
       || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
      ){
        int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
-        if( rc!=SQLITE_OK ){
-          pNode->bNomatch = 0;
-          return rc;
-        }
+        if( rc!=SQLITE_OK ) return rc;
      }
    }
  }
@@ -1202,10 +1128,7 @@ static int fts5ExprNodeTest_AND(
      if( cmp>0 ){
        /* Advance pChild until it points to iLast or laster */
        rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
-        if( rc!=SQLITE_OK ){
-          pAnd->bNomatch = 0;
-          return rc;
-        }
+        if( rc!=SQLITE_OK ) return rc;
      }

      /* If the child node is now at EOF, so is the parent AND node. Otherwise,
@@ -1244,8 +1167,6 @@ static int fts5ExprNodeNext_AND(
  int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
  if( rc==SQLITE_OK ){
    rc = fts5ExprNodeTest_AND(pExpr, pNode);
-  }else{
-    pNode->bNomatch = 0;
  }
  return rc;
 }
@@ -1288,9 +1209,6 @@ static int fts5ExprNodeNext_NOT(
  if( rc==SQLITE_OK ){
    rc = fts5ExprNodeTest_NOT(pExpr, pNode);
  }
-  if( rc!=SQLITE_OK ){
-    pNode->bNomatch = 0;
-  }
  return rc;
 }

@@ -1413,10 +1331,7 @@ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){

  /* If not at EOF but the current rowid occurs earlier than iFirst in
  ** the iteration order, move to document iFirst or later. */
-  if( rc==SQLITE_OK 
-   && 0==pRoot->bEof 
-   && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 
-  ){
+  if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
    rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
  }

@@ -1486,16 +1401,6 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
  }
 }

-/*
-** Set the "bFirst" flag on the first token of the phrase passed as the
-** only argument.
-*/
-void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){
-  if( pPhrase && pPhrase->nTerm ){
-    pPhrase->aTerm[0].bFirst = 1;
-  }
-}
-
 /*
 ** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
 ** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
@@ -1517,20 +1422,18 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
      return pNear;
    }
    if( pNear==0 ){
-      sqlite3_int64 nByte;
-      nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
-      pRet = sqlite3_malloc64(nByte);
+      int nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
+      pRet = sqlite3_malloc(nByte);
      if( pRet==0 ){
        pParse->rc = SQLITE_NOMEM;
      }else{
-        memset(pRet, 0, (size_t)nByte);
+        memset(pRet, 0, nByte);
      }
    }else if( (pNear->nPhrase % SZALLOC)==0 ){
      int nNew = pNear->nPhrase + SZALLOC;
-      sqlite3_int64 nByte;
+      int nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*);

-      nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*);
-      pRet = (Fts5ExprNearset*)sqlite3_realloc64(pNear, nByte);
+      pRet = (Fts5ExprNearset*)sqlite3_realloc(pNear, nByte);
      if( pRet==0 ){
        pParse->rc = SQLITE_NOMEM;
      }
@@ -1594,12 +1497,12 @@ static int fts5ParseTokenize(

  if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){
    Fts5ExprTerm *pSyn;
-    sqlite3_int64 nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
-    pSyn = (Fts5ExprTerm*)sqlite3_malloc64(nByte);
+    int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
+    pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
    if( pSyn==0 ){
      rc = SQLITE_NOMEM;
    }else{
-      memset(pSyn, 0, (size_t)nByte);
+      memset(pSyn, 0, nByte);
      pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
      memcpy(pSyn->zTerm, pToken, nToken);
      pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
@@ -1611,7 +1514,7 @@ static int fts5ParseTokenize(
      Fts5ExprPhrase *pNew;
      int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);

-      pNew = (Fts5ExprPhrase*)sqlite3_realloc64(pPhrase, 
+      pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, 
          sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
      );
      if( pNew==0 ){
@@ -1682,7 +1585,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(

  rc = fts5ParseStringFromToken(pToken, &z);
  if( rc==SQLITE_OK ){
-    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
+    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
    int n;
    sqlite3Fts5Dequote(z);
    n = (int)strlen(z);
@@ -1697,9 +1600,9 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(

    if( pAppend==0 ){
      if( (pParse->nPhrase % 8)==0 ){
-        sqlite3_int64 nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
+        int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
        Fts5ExprPhrase **apNew;
-        apNew = (Fts5ExprPhrase**)sqlite3_realloc64(pParse->apPhrase, nByte);
+        apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
        if( apNew==0 ){
          pParse->rc = SQLITE_NOMEM;
          fts5ExprPhraseFree(sCtx.pPhrase);
@@ -1715,7 +1618,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
      ** no token characters at all. (e.g ... MATCH '""'). */
      sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase));
    }else if( sCtx.pPhrase->nTerm ){
-      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
+      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
    }
    pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
  }
@@ -1754,12 +1657,10 @@ int sqlite3Fts5ExprClonePhrase(
  if( rc==SQLITE_OK ){
    Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset;
    if( pColsetOrig ){
-      sqlite3_int64 nByte;
-      Fts5Colset *pColset;
-      nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int);
-      pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte);
+      int nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int);
+      Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte);
      if( pColset ){ 
-        memcpy(pColset, pColsetOrig, (size_t)nByte);
+        memcpy(pColset, pColsetOrig, nByte);
      }
      pNew->pRoot->pNear->pColset = pColset;
    }
@@ -1778,7 +1679,6 @@ int sqlite3Fts5ExprClonePhrase(
      }
      if( rc==SQLITE_OK ){
        sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
-        sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
      }
    }
  }else{
@@ -1797,10 +1697,7 @@ int sqlite3Fts5ExprClonePhrase(
    pNew->pRoot->pNear->nPhrase = 1;
    sCtx.pPhrase->pNode = pNew->pRoot;

-    if( pOrig->nTerm==1 
-     && pOrig->aTerm[0].pSynonym==0 
-     && pOrig->aTerm[0].bFirst==0 
-    ){
+    if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
      pNew->pRoot->eType = FTS5_TERM;
      pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
    }else{
@@ -1877,7 +1774,7 @@ static Fts5Colset *fts5ParseColset(
  assert( pParse->rc==SQLITE_OK );
  assert( iCol>=0 && iCol<pParse->pConfig->nCol );

-  pNew = sqlite3_realloc64(p, sizeof(Fts5Colset) + sizeof(int)*nCol);
+  pNew = sqlite3_realloc(p, sizeof(Fts5Colset) + sizeof(int)*nCol);
  if( pNew==0 ){
    pParse->rc = SQLITE_NOMEM;
  }else{
@@ -1962,110 +1859,25 @@ Fts5Colset *sqlite3Fts5ParseColset(
  return pRet;
 }

-/*
-** If argument pOrig is NULL, or if (*pRc) is set to anything other than
-** SQLITE_OK when this function is called, NULL is returned. 
-**
-** Otherwise, a copy of (*pOrig) is made into memory obtained from
-** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
-** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
-*/
-static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
-  Fts5Colset *pRet;
-  if( pOrig ){
-    sqlite3_int64 nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
-    pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
-    if( pRet ){ 
-      memcpy(pRet, pOrig, (size_t)nByte);
-    }
-  }else{
-    pRet = 0;
-  }
-  return pRet;
-}
-
-/*
-** Remove from colset pColset any columns that are not also in colset pMerge.
-*/
-static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
-  int iIn = 0;          /* Next input in pColset */
-  int iMerge = 0;       /* Next input in pMerge */
-  int iOut = 0;         /* Next output slot in pColset */
-
-  while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
-    int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
-    if( iDiff==0 ){
-      pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
-      iMerge++;
-      iIn++;
-    }else if( iDiff>0 ){
-      iMerge++;
-    }else{
-      iIn++;
-    }
-  }
-  pColset->nCol = iOut;
-}
-
-/*
-** Recursively apply colset pColset to expression node pNode and all of
-** its decendents. If (*ppFree) is not NULL, it contains a spare copy
-** of pColset. This function may use the spare copy and set (*ppFree) to
-** zero, or it may create copies of pColset using fts5CloneColset().
-*/
-static void fts5ParseSetColset(
-  Fts5Parse *pParse, 
-  Fts5ExprNode *pNode, 
-  Fts5Colset *pColset,
-  Fts5Colset **ppFree
-){
-  if( pParse->rc==SQLITE_OK ){
-    assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING 
-         || pNode->eType==FTS5_AND  || pNode->eType==FTS5_OR
-         || pNode->eType==FTS5_NOT  || pNode->eType==FTS5_EOF
-    );
-    if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
-      Fts5ExprNearset *pNear = pNode->pNear;
-      if( pNear->pColset ){
-        fts5MergeColset(pNear->pColset, pColset);
-        if( pNear->pColset->nCol==0 ){
-          pNode->eType = FTS5_EOF;
-          pNode->xNext = 0;
-        }
-      }else if( *ppFree ){
-        pNear->pColset = pColset;
-        *ppFree = 0;
-      }else{
-        pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
-      }
-    }else{
-      int i;
-      assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
-      for(i=0; i<pNode->nChild; i++){
-        fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
-      }
-    }
-  }
-}
-
-/*
-** Apply colset pColset to expression node pExpr and all of its descendents.
-*/
 void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
-  Fts5ExprNode *pExpr, 
+  Fts5ExprNearset *pNear, 
  Fts5Colset *pColset 
 ){
-  Fts5Colset *pFree = pColset;
  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
    pParse->rc = SQLITE_ERROR;
    pParse->zErr = sqlite3_mprintf(
      "fts5: column queries are not supported (detail=none)"
    );
-  }else{
-    fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
+    sqlite3_free(pColset);
+    return;
+  }
+
+  if( pNear ){
+    pNear->pColset = pColset;
+  }else{
+    sqlite3_free(pColset);
  }
-  sqlite3_free(pFree);
 }

 static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
@@ -2074,7 +1886,6 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
      Fts5ExprNearset *pNear = pNode->pNear;
      if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 
       && pNear->apPhrase[0]->aTerm[0].pSynonym==0
-       && pNear->apPhrase[0]->aTerm[0].bFirst==0
      ){
        pNode->eType = FTS5_TERM;
        pNode->xNext = fts5ExprNodeNext_TERM;
@@ -2127,7 +1938,7 @@ Fts5ExprNode *sqlite3Fts5ParseNode(

  if( pParse->rc==SQLITE_OK ){
    int nChild = 0;               /* Number of children of returned node */
-    sqlite3_int64 nByte;          /* Bytes of space to allocate for this node */
+    int nByte;                    /* Bytes of space to allocate for this node */
 
    assert( (eType!=FTS5_STRING && !pNear)
         || (eType==FTS5_STRING && !pLeft && !pRight)
@@ -2161,23 +1972,20 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
          }
        }

-        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
-          Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
-          if( pNear->nPhrase!=1 
-           || pPhrase->nTerm>1
-           || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst)
-          ){
-            assert( pParse->rc==SQLITE_OK );
-            pParse->rc = SQLITE_ERROR;
-            assert( pParse->zErr==0 );
-            pParse->zErr = sqlite3_mprintf(
-                "fts5: %s queries are not supported (detail!=full)", 
-                pNear->nPhrase==1 ? "phrase": "NEAR"
-                );
-            sqlite3_free(pRet);
-            pRet = 0;
-          }
+        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL 
+         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
+        ){
+          assert( pParse->rc==SQLITE_OK );
+          pParse->rc = SQLITE_ERROR;
+          assert( pParse->zErr==0 );
+          pParse->zErr = sqlite3_mprintf(
+              "fts5: %s queries are not supported (detail!=full)", 
+              pNear->nPhrase==1 ? "phrase": "NEAR"
+          );
+          sqlite3_free(pRet);
+          pRet = 0;
        }
+
      }else{
        fts5ExprAddChildren(pRet, pLeft);
        fts5ExprAddChildren(pRet, pRight);
@@ -2259,7 +2067,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
 }

 static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
-  sqlite3_int64 nByte = 0;
+  int nByte = 0;
  Fts5ExprTerm *p;
  char *zQuoted;

@@ -2267,7 +2075,7 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
  for(p=pTerm; p; p=p->pSynonym){
    nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2;
  }
-  zQuoted = sqlite3_malloc64(nByte);
+  zQuoted = sqlite3_malloc(nByte);

  if( zQuoted ){
    int i = 0;
@@ -2507,7 +2315,7 @@ static void fts5ExprFunction(
  }

  nConfig = 3 + (nArg-iArg);
-  azConfig = (const char**)sqlite3_malloc64(sizeof(char*) * nConfig);
+  azConfig = (const char**)sqlite3_malloc(sizeof(char*) * nConfig);
  if( azConfig==0 ){
    sqlite3_result_error_nomem(pCtx);
    return;
@@ -2516,16 +2324,14 @@ static void fts5ExprFunction(
  azConfig[1] = "main";
  azConfig[2] = "tbl";
  for(i=3; iArg<nArg; iArg++){
-    const char *z = (const char*)sqlite3_value_text(apVal[iArg]);
-    azConfig[i++] = (z ? z : "");
+    azConfig[i++] = (const char*)sqlite3_value_text(apVal[iArg]);
  }

  zExpr = (const char*)sqlite3_value_text(apVal[0]);
-  if( zExpr==0 ) zExpr = "";

  rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  if( rc==SQLITE_OK ){
-    rc = sqlite3Fts5ExprNew(pConfig, pConfig->nCol, zExpr, &pExpr, &zErr);
+    rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
  }
  if( rc==SQLITE_OK ){
    char *zText;
@@ -2583,19 +2389,14 @@ static void fts5ExprIsAlnum(
  sqlite3_value **apVal           /* Function arguments */
 ){
  int iCode;
-  u8 aArr[32];
  if( nArg!=1 ){
    sqlite3_result_error(pCtx, 
        "wrong number of arguments to function fts5_isalnum", -1
    );
    return;
  }
-  memset(aArr, 0, sizeof(aArr));
-  sqlite3Fts5UnicodeCatParse("L*", aArr);
-  sqlite3Fts5UnicodeCatParse("N*", aArr);
-  sqlite3Fts5UnicodeCatParse("Co", aArr);
  iCode = sqlite3_value_int(apVal[0]);
-  sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory((u32)iCode)]);
+  sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode));
 }

 static void fts5ExprFold(
@@ -2639,12 +2440,10 @@ int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
    rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
  }

-  /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and
-  ** sqlite3Fts5ParserFallback() are unused */
+  /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */
 #ifndef NDEBUG
  (void)sqlite3Fts5ParserTrace;
 #endif
-  (void)sqlite3Fts5ParserFallback;

  return rc;
 }
@@ -2690,7 +2489,7 @@ struct Fts5PoslistPopulator {

 Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){
  Fts5PoslistPopulator *pRet;
-  pRet = sqlite3_malloc64(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
+  pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
  if( pRet ){
    int i;
    memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
@@ -2889,3 +2688,4 @@ int sqlite3Fts5ExprPhraseCollist(

  return rc;
 }
+
@@ -36,10 +36,9 @@ struct Fts5Hash {

 /*
 ** Each entry in the hash table is represented by an object of the 
-** following type. Each object, its key (a nul-terminated string) and 
-** its current data are stored in a single memory allocation. The 
-** key immediately follows the object in memory. The position list
-** data immediately follows the key data in memory.
+** following type. Each object, its key (zKey[]) and its current data
+** are stored in a single memory allocation. The position list data 
+** immediately follows the key data in memory.
 **
 ** The data that follows the key is in a similar, but not identical format
 ** to the doclist data stored in the database. It is:
@@ -63,20 +62,20 @@ struct Fts5HashEntry {
  int nAlloc;                     /* Total size of allocation */
  int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
  int nData;                      /* Total bytes of data (incl. structure) */
-  int nKey;                       /* Length of key in bytes */
+  int nKey;                       /* Length of zKey[] in bytes */
  u8 bDel;                        /* Set delete-flag @ iSzPoslist */
  u8 bContent;                    /* Set content-flag (detail=none mode) */
  i16 iCol;                       /* Column of last value written */
  int iPos;                       /* Position of last value written */
  i64 iRowid;                     /* Rowid of last value written */
+  char zKey[8];                   /* Nul-terminated entry key */
 };

 /*
-** Eqivalent to:
-**
-**   char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; }
+** Size of Fts5HashEntry without the zKey[] array.
 */
-#define fts5EntryKey(p) ( ((char *)(&(p)[1])) )
+#define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8)
+


 /*
@@ -90,20 +89,20 @@ int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
  if( pNew==0 ){
    rc = SQLITE_NOMEM;
  }else{
-    sqlite3_int64 nByte;
+    int nByte;
    memset(pNew, 0, sizeof(Fts5Hash));
    pNew->pnByte = pnByte;
    pNew->eDetail = pConfig->eDetail;

    pNew->nSlot = 1024;
    nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
-    pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc64(nByte);
+    pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte);
    if( pNew->aSlot==0 ){
      sqlite3_free(pNew);
      *ppNew = 0;
      rc = SQLITE_NOMEM;
    }else{
-      memset(pNew->aSlot, 0, (size_t)nByte);
+      memset(pNew->aSlot, 0, nByte);
    }
  }
  return rc;
@@ -165,17 +164,16 @@ static int fts5HashResize(Fts5Hash *pHash){
  Fts5HashEntry **apNew;
  Fts5HashEntry **apOld = pHash->aSlot;

-  apNew = (Fts5HashEntry**)sqlite3_malloc64(nNew*sizeof(Fts5HashEntry*));
+  apNew = (Fts5HashEntry**)sqlite3_malloc(nNew*sizeof(Fts5HashEntry*));
  if( !apNew ) return SQLITE_NOMEM;
  memset(apNew, 0, nNew*sizeof(Fts5HashEntry*));

  for(i=0; i<pHash->nSlot; i++){
    while( apOld[i] ){
-      unsigned int iHash;
+      int iHash;
      Fts5HashEntry *p = apOld[i];
      apOld[i] = p->pHashNext;
-      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
-                          (int)strlen(fts5EntryKey(p)));
+      iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey));
      p->pHashNext = apNew[iHash];
      apNew[iHash] = p;
    }
@@ -187,25 +185,19 @@ static int fts5HashResize(Fts5Hash *pHash){
  return SQLITE_OK;
 }

-static int fts5HashAddPoslistSize(
-  Fts5Hash *pHash, 
-  Fts5HashEntry *p,
-  Fts5HashEntry *p2
-){
-  int nRet = 0;
+static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
  if( p->iSzPoslist ){
-    u8 *pPtr = p2 ? (u8*)p2 : (u8*)p;
-    int nData = p->nData;
+    u8 *pPtr = (u8*)p;
    if( pHash->eDetail==FTS5_DETAIL_NONE ){
-      assert( nData==p->iSzPoslist );
+      assert( p->nData==p->iSzPoslist );
      if( p->bDel ){
-        pPtr[nData++] = 0x00;
+        pPtr[p->nData++] = 0x00;
        if( p->bContent ){
-          pPtr[nData++] = 0x00;
+          pPtr[p->nData++] = 0x00;
        }
      }
    }else{
-      int nSz = (nData - p->iSzPoslist - 1);       /* Size in bytes */
+      int nSz = (p->nData - p->iSzPoslist - 1);       /* Size in bytes */
      int nPos = nSz*2 + p->bDel;                     /* Value of nPos field */

      assert( p->bDel==0 || p->bDel==1 );
@@ -215,19 +207,14 @@ static int fts5HashAddPoslistSize(
        int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
        memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
        sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
-        nData += (nByte-1);
+        p->nData += (nByte-1);
      }
    }

-    nRet = nData - p->nData;
-    if( p2==0 ){
-      p->iSzPoslist = 0;
-      p->bDel = 0;
-      p->bContent = 0;
-      p->nData = nData;
-    }
+    p->iSzPoslist = 0;
+    p->bDel = 0;
+    p->bContent = 0;
  }
-  return nRet;
 }

 /*
@@ -257,10 +244,9 @@ int sqlite3Fts5HashWrite(
  /* Attempt to locate an existing hash entry */
  iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
-    char *zKey = fts5EntryKey(p);
-    if( zKey[0]==bByte 
+    if( p->zKey[0]==bByte 
     && p->nKey==nToken
-     && memcmp(&zKey[1], pToken, nToken)==0 
+     && memcmp(&p->zKey[1], pToken, nToken)==0 
    ){
      break;
    }
@@ -269,8 +255,7 @@ int sqlite3Fts5HashWrite(
  /* If an existing hash entry cannot be found, create a new one. */
  if( p==0 ){
    /* Figure out how much space to allocate */
-    char *zKey;
-    sqlite3_int64 nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64;
+    int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
    if( nByte<128 ) nByte = 128;

    /* Grow the Fts5Hash.aSlot[] array if necessary. */
@@ -281,17 +266,16 @@ int sqlite3Fts5HashWrite(
    }

    /* Allocate new Fts5HashEntry and add it to the hash table. */
-    p = (Fts5HashEntry*)sqlite3_malloc64(nByte);
+    p = (Fts5HashEntry*)sqlite3_malloc(nByte);
    if( !p ) return SQLITE_NOMEM;
-    memset(p, 0, sizeof(Fts5HashEntry));
-    p->nAlloc = (int)nByte;
-    zKey = fts5EntryKey(p);
-    zKey[0] = bByte;
-    memcpy(&zKey[1], pToken, nToken);
-    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
+    memset(p, 0, FTS5_HASHENTRYSIZE);
+    p->nAlloc = nByte;
+    p->zKey[0] = bByte;
+    memcpy(&p->zKey[1], pToken, nToken);
+    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
    p->nKey = nToken;
-    zKey[nToken+1] = '\0';
-    p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
+    p->zKey[nToken+1] = '\0';
+    p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
    p->pHashNext = pHash->aSlot[iHash];
    pHash->aSlot[iHash] = p;
    pHash->nEntry++;
@@ -320,12 +304,12 @@ int sqlite3Fts5HashWrite(
    **     + 5 bytes for the new position offset (32-bit max).
    */
    if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
-      sqlite3_int64 nNew = p->nAlloc * 2;
+      int nNew = p->nAlloc * 2;
      Fts5HashEntry *pNew;
      Fts5HashEntry **pp;
-      pNew = (Fts5HashEntry*)sqlite3_realloc64(p, nNew);
+      pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
      if( pNew==0 ) return SQLITE_NOMEM;
-      pNew->nAlloc = (int)nNew;
+      pNew->nAlloc = nNew;
      for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
      *pp = pNew;
      p = pNew;
@@ -339,7 +323,7 @@ int sqlite3Fts5HashWrite(
  /* If this is a new rowid, append the 4-byte size field for the previous
  ** entry, and the new rowid for this entry.  */
  if( iRowid!=p->iRowid ){
-    fts5HashAddPoslistSize(pHash, p, 0);
+    fts5HashAddPoslistSize(pHash, p);
    p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
    p->iRowid = iRowid;
    bNew = 1;
@@ -409,11 +393,9 @@ static Fts5HashEntry *fts5HashEntryMerge(
      p1 = 0;
    }else{
      int i = 0;
-      char *zKey1 = fts5EntryKey(p1);
-      char *zKey2 = fts5EntryKey(p2);
-      while( zKey1[i]==zKey2[i] ) i++;
+      while( p1->zKey[i]==p2->zKey[i] ) i++;

-      if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
+      if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){
        /* p2 is smaller */
        *ppOut = p2;
        ppOut = &p2->pScanNext;
@@ -449,16 +431,14 @@ static int fts5HashEntrySort(
  int i;

  *ppSorted = 0;
-  ap = sqlite3_malloc64(sizeof(Fts5HashEntry*) * nMergeSlot);
+  ap = sqlite3_malloc(sizeof(Fts5HashEntry*) * nMergeSlot);
  if( !ap ) return SQLITE_NOMEM;
  memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot);

  for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
    Fts5HashEntry *pIter;
    for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
-      if( pTerm==0 
-       || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
-      ){
+      if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){
        Fts5HashEntry *pEntry = pIter;
        pEntry->pScanNext = 0;
        for(i=0; ap[i]; i++){
@@ -486,36 +466,23 @@ static int fts5HashEntrySort(
 */
 int sqlite3Fts5HashQuery(
  Fts5Hash *pHash,                /* Hash table to query */
-  int nPre,
  const char *pTerm, int nTerm,   /* Query term */
-  void **ppOut,                   /* OUT: Pointer to new object */
+  const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
  int *pnDoclist                  /* OUT: Size of doclist in bytes */
 ){
  unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
-  char *zKey = 0;
  Fts5HashEntry *p;

  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
-    zKey = fts5EntryKey(p);
-    assert( p->nKey+1==(int)strlen(zKey) );
-    if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break;
+    if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break;
  }

  if( p ){
-    int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1;
-    int nList = p->nData - nHashPre;
-    u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10));
-    if( pRet ){
-      Fts5HashEntry *pFaux = (Fts5HashEntry*)&pRet[nPre-nHashPre];
-      memcpy(&pRet[nPre], &((u8*)p)[nHashPre], nList);
-      nList += fts5HashAddPoslistSize(pHash, p, pFaux);
-      *pnDoclist = nList;
-    }else{
-      *pnDoclist = 0;
-      return SQLITE_NOMEM;
-    }
+    fts5HashAddPoslistSize(pHash, p);
+    *ppDoclist = (const u8*)&p->zKey[nTerm+1];
+    *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
  }else{
-    *ppOut = 0;
+    *ppDoclist = 0;
    *pnDoclist = 0;
  }

@@ -546,15 +513,15 @@ void sqlite3Fts5HashScanEntry(
 ){
  Fts5HashEntry *p;
  if( (p = pHash->pScan) ){
-    char *zKey = fts5EntryKey(p);
-    int nTerm = (int)strlen(zKey);
-    fts5HashAddPoslistSize(pHash, p, 0);
-    *pzTerm = zKey;
-    *ppDoclist = (const u8*)&zKey[nTerm+1];
-    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
+    int nTerm = (int)strlen(p->zKey);
+    fts5HashAddPoslistSize(pHash, p);
+    *pzTerm = p->zKey;
+    *ppDoclist = (const u8*)&p->zKey[nTerm+1];
+    *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
  }else{
    *pzTerm = 0;
    *ppDoclist = 0;
    *pnDoclist = 0;
  }
 }
+
@@ -115,7 +115,7 @@ static int fts5StorageGetStmt(
        char *zBind;
        int i;

-        zBind = sqlite3_malloc64(1 + nCol*2);
+        zBind = sqlite3_malloc(1 + nCol*2);
        if( zBind ){
          for(i=0; i<nCol; i++){
            zBind[i*2] = '?';
@@ -136,11 +136,7 @@ static int fts5StorageGetStmt(
    if( zSql==0 ){
      rc = SQLITE_NOMEM;
    }else{
-      int f = SQLITE_PREPARE_PERSISTENT;
-      if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB;
-      p->pConfig->bLock++;
-      rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
-      p->pConfig->bLock--;
+      rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
      sqlite3_free(zSql);
      if( rc!=SQLITE_OK && pzErrMsg ){
        *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
@@ -222,7 +218,7 @@ static void fts5StorageRenameOne(

 int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
  Fts5Config *pConfig = pStorage->pConfig;
-  int rc = sqlite3Fts5StorageSync(pStorage);
+  int rc = sqlite3Fts5StorageSync(pStorage, 1);

  fts5StorageRenameOne(pConfig, &rc, "data", zName);
  fts5StorageRenameOne(pConfig, &rc, "idx", zName);
@@ -284,14 +280,14 @@ int sqlite3Fts5StorageOpen(
 ){
  int rc = SQLITE_OK;
  Fts5Storage *p;                 /* New object */
-  sqlite3_int64 nByte;            /* Bytes of space to allocate */
+  int nByte;                      /* Bytes of space to allocate */

  nByte = sizeof(Fts5Storage)               /* Fts5Storage object */
        + pConfig->nCol * sizeof(i64);      /* Fts5Storage.aTotalSize[] */
-  *pp = p = (Fts5Storage*)sqlite3_malloc64(nByte);
+  *pp = p = (Fts5Storage*)sqlite3_malloc(nByte);
  if( !p ) return SQLITE_NOMEM;

-  memset(p, 0, (size_t)nByte);
+  memset(p, 0, nByte);
  p->aTotalSize = (i64*)&p[1];
  p->pConfig = pConfig;
  p->pIndex = pIndex;
@@ -299,7 +295,7 @@ int sqlite3Fts5StorageOpen(
  if( bCreate ){
    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
      int nDefn = 32 + pConfig->nCol*10;
-      char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10);
+      char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
      if( zDefn==0 ){
        rc = SQLITE_NOMEM;
      }else{
@@ -461,7 +457,6 @@ static int fts5StorageInsertDocsize(
      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
      sqlite3_step(pReplace);
      rc = sqlite3_reset(pReplace);
-      sqlite3_bind_null(pReplace, 2);
    }
  }
  return rc;
@@ -550,6 +545,11 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
    }
  }

+  /* Write the averages record */
+  if( rc==SQLITE_OK ){
+    rc = fts5StorageSaveTotals(p);
+  }
+
  return rc;
 }

@@ -560,8 +560,6 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
  Fts5Config *pConfig = p->pConfig;
  int rc;

-  p->bTotalsValid = 0;
-
  /* Delete the contents of the %_data and %_docsize tables. */
  rc = fts5ExecPrintf(pConfig->db, 0,
      "DELETE FROM %Q.'%q_data';" 
@@ -592,7 +590,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
  Fts5Config *pConfig = p->pConfig;
  sqlite3_stmt *pScan = 0;
  Fts5InsertCtx ctx;
-  int rc, rc2;
+  int rc;

  memset(&ctx, 0, sizeof(Fts5InsertCtx));
  ctx.pStorage = p;
@@ -613,11 +611,10 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
    for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
      ctx.szCol = 0;
      if( pConfig->abUnindexed[ctx.iCol]==0 ){
-        const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1);
-        int nText = sqlite3_column_bytes(pScan, ctx.iCol+1);
        rc = sqlite3Fts5Tokenize(pConfig, 
            FTS5_TOKENIZE_DOCUMENT,
-            zText, nText,
+            (const char*)sqlite3_column_text(pScan, ctx.iCol+1),
+            sqlite3_column_bytes(pScan, ctx.iCol+1),
            (void*)&ctx,
            fts5StorageInsertCallback
        );
@@ -632,8 +629,6 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
    }
  }
  sqlite3_free(buf.p);
-  rc2 = sqlite3_reset(pScan);
-  if( rc==SQLITE_OK ) rc = rc2;

  /* Write the averages record */
  if( rc==SQLITE_OK ){
@@ -739,11 +734,10 @@ int sqlite3Fts5StorageIndexInsert(
  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
    ctx.szCol = 0;
    if( pConfig->abUnindexed[ctx.iCol]==0 ){
-      const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
-      int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
      rc = sqlite3Fts5Tokenize(pConfig, 
          FTS5_TOKENIZE_DOCUMENT,
-          zText, nText,
+          (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
+          sqlite3_value_bytes(apVal[ctx.iCol+2]),
          (void*)&ctx,
          fts5StorageInsertCallback
      );
@@ -759,6 +753,11 @@ int sqlite3Fts5StorageIndexInsert(
  }
  sqlite3_free(buf.p);

+  /* Write the averages record */
+  if( rc==SQLITE_OK ){
+    rc = fts5StorageSaveTotals(p);
+  }
+
  return rc;
 }

@@ -884,7 +883,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){

  memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
  ctx.pConfig = p->pConfig;
-  aTotalSize = (i64*)sqlite3_malloc64(pConfig->nCol*(sizeof(int)+sizeof(i64)));
+  aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64)));
  if( !aTotalSize ) return SQLITE_NOMEM;
  aColSize = (int*)&aTotalSize[pConfig->nCol];
  memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
@@ -912,11 +911,10 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
          rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
        }
        if( rc==SQLITE_OK ){
-          const char *zText = (const char*)sqlite3_column_text(pScan, i+1);
-          int nText = sqlite3_column_bytes(pScan, i+1);
          rc = sqlite3Fts5Tokenize(pConfig, 
              FTS5_TOKENIZE_DOCUMENT,
-              zText, nText,
+              (const char*)sqlite3_column_text(pScan, i+1),
+              sqlite3_column_bytes(pScan, i+1),
              (void*)&ctx,
              fts5StorageIntegrityCallback
          );
@@ -1085,13 +1083,7 @@ int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
  int rc = fts5StorageLoadTotals(p, 0);
  if( rc==SQLITE_OK ){
-    /* nTotalRow being zero does not necessarily indicate a corrupt 
-    ** database - it might be that the FTS5 table really does contain zero
-    ** rows. However this function is only called from the xRowCount() API,
-    ** and there is no way for that API to be invoked if the table contains
-    ** no rows. Hence the FTS5_CORRUPT return.  */
    *pnRow = p->nTotalRow;
-    if( p->nTotalRow<=0 ) rc = FTS5_CORRUPT;
  }
  return rc;
 }
@@ -1099,18 +1091,13 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
 /*
 ** Flush any data currently held in-memory to disk.
 */
-int sqlite3Fts5StorageSync(Fts5Storage *p){
-  int rc = SQLITE_OK;
-  i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
-  if( p->bTotalsValid ){
-    rc = fts5StorageSaveTotals(p);
+int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){
+  if( bCommit && p->bTotalsValid ){
+    int rc = fts5StorageSaveTotals(p);
    p->bTotalsValid = 0;
+    if( rc!=SQLITE_OK ) return rc;
  }
-  if( rc==SQLITE_OK ){
-    rc = sqlite3Fts5IndexSync(p->pIndex);
-  }
-  sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid);
-  return rc;
+  return sqlite3Fts5IndexSync(p->pIndex, bCommit);
 }

 int sqlite3Fts5StorageRollback(Fts5Storage *p){
@@ -1135,7 +1122,6 @@ int sqlite3Fts5StorageConfigValue(
    }
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
-    sqlite3_bind_null(pReplace, 1);
  }
  if( rc==SQLITE_OK && pVal ){
    int iNew = p->pConfig->iCookie + 1;
@@ -99,13 +99,16 @@ static int SQLITE_TCLAPI f5tDbAndApi(
    sqlite3_stmt *pStmt = 0;
    fts5_api *pApi = 0;

-    rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);
+    rc = sqlite3_prepare_v2(db, "SELECT fts5()", -1, &pStmt, 0);
    if( rc!=SQLITE_OK ){
      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
      return TCL_ERROR;
    }
-    sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0);
-    sqlite3_step(pStmt);
+
+    if( SQLITE_ROW==sqlite3_step(pStmt) ){
+      const void *pPtr = sqlite3_column_blob(pStmt, 0);
+      memcpy((void*)&pApi, pPtr, sizeof(pApi));
+    }

    if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
@@ -433,7 +436,7 @@ static int SQLITE_TCLAPI xF5tApi(
      int iVal;
      int bClear;
      if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
-      iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
+      iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
      Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
      break;
    }
@@ -482,7 +485,7 @@ static int SQLITE_TCLAPI xF5tApi(

      rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
      if( rc!=SQLITE_OK ){
-        Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
+        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
        return TCL_ERROR;
      }
      for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
@@ -924,7 +927,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(

  rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
  Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
-  return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
+  return TCL_OK;

 usage:
  Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END");
@@ -73,10 +73,13 @@ static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
  int rc;

  *ppApi = 0;
-  rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0);
+  rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0);
  if( rc==SQLITE_OK ){
-    sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0);
-    (void)sqlite3_step(pStmt);
+    if( SQLITE_ROW==sqlite3_step(pStmt) 
+        && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0)
+      ){
+      memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*));
+    }
    rc = sqlite3_finalize(pStmt);
  }

@@ -309,7 +312,7 @@ static Fts5MatchinfoCtx *fts5MatchinfoNew(
  int nPhrase;
  int i;
  int nInt;
-  sqlite3_int64 nByte;
+  int nByte;
  int rc;

  nCol = pApi->xColumnCount(pFts);
@@ -330,7 +333,7 @@ static Fts5MatchinfoCtx *fts5MatchinfoNew(
  nByte = sizeof(Fts5MatchinfoCtx)          /* The struct itself */
         + sizeof(u32) * nInt               /* The p->aRet[] array */
         + (i+1);                           /* The p->zArg string */
-  p = (Fts5MatchinfoCtx*)sqlite3_malloc64(nByte);
+  p = (Fts5MatchinfoCtx*)sqlite3_malloc(nByte);
  if( p==0 ){
    sqlite3_result_error_nomem(pCtx);
    return 0;
@@ -419,3 +422,4 @@ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){
 }

 #endif /* SQLITE_ENABLE_FTS5 */
+
@@ -40,7 +40,7 @@
 */
 #if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)

-#include "fts5.h"
+#include <fts5.h>
 #include <string.h>
 #include <assert.h>

@@ -137,7 +137,7 @@ static int fts5tokDequoteArray(
      nByte += (int)(strlen(argv[i]) + 1);
    }

-    *pazDequote = azDequote = sqlite3_malloc64(sizeof(char *)*argc + nByte);
+    *pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
    if( azDequote==0 ){
      rc = SQLITE_NOMEM;
    }else{
@@ -182,7 +182,7 @@ static int fts5tokConnectMethod(
  Fts5tokTable *pTab = 0;
  int rc;
  char **azDequote = 0;
-  int nDequote = 0;
+  int nDequote;

  rc = sqlite3_declare_vtab(db, 
       "CREATE TABLE x(input HIDDEN, token, start, end, position)"
@@ -335,7 +335,7 @@ static int fts5tokCb(
  if( (pCsr->nRow & (pCsr->nRow-1))==0 ){
    int nNew = pCsr->nRow ? pCsr->nRow*2 : 32;
    Fts5tokRow *aNew;
-    aNew = (Fts5tokRow*)sqlite3_realloc64(pCsr->aRow, nNew*sizeof(Fts5tokRow));
+    aNew = (Fts5tokRow*)sqlite3_realloc(pCsr->aRow, nNew*sizeof(Fts5tokRow));
    if( aNew==0 ) return SQLITE_NOMEM;
    memset(&aNew[pCsr->nRow], 0, sizeof(Fts5tokRow)*(nNew-pCsr->nRow));
    pCsr->aRow = aNew;
@@ -378,7 +378,7 @@ static int fts5tokFilterMethod(
    if( pCsr->zInput==0 ){
      rc = SQLITE_NOMEM;
    }else{
-      if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte);
+      memcpy(pCsr->zInput, zByte, nByte);
      pCsr->zInput[nByte] = 0;
      rc = pTab->tok.xTokenize(
          pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb
@@ -471,8 +471,7 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
     0,                           /* xRename       */
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
-     0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0                            /* xRollbackTo   */
  };
  int rc;                         /* Return code */

@@ -152,7 +152,7 @@ static int fts5AsciiTokenize(
    nByte = ie-is;
    if( nByte>nFold ){
      if( pFold!=aFold ) sqlite3_free(pFold);
-      pFold = sqlite3_malloc64((sqlite3_int64)nByte*2);
+      pFold = sqlite3_malloc(nByte*2);
      if( pFold==0 ){
        rc = SQLITE_NOMEM;
        break;
@@ -234,18 +234,11 @@ struct Unicode61Tokenizer {
  unsigned char aTokenChar[128];  /* ASCII range token characters */
  char *aFold;                    /* Buffer to fold text into */
  int nFold;                      /* Size of aFold[] in bytes */
-  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
+  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
  int nException;
  int *aiException;
-
-  unsigned char aCategory[32];    /* True for token char categories */
 };

-/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
-#define FTS5_REMOVE_DIACRITICS_NONE    0
-#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
-#define FTS5_REMOVE_DIACRITICS_COMPLEX 2
-
 static int fts5UnicodeAddExceptions(
  Unicode61Tokenizer *p,          /* Tokenizer object */
  const char *z,                  /* Characters to treat as exceptions */
@@ -256,26 +249,25 @@ static int fts5UnicodeAddExceptions(
  int *aNew;

  if( n>0 ){
-    aNew = (int*)sqlite3_realloc64(p->aiException,
-                                   (n+p->nException)*sizeof(int));
+    aNew = (int*)sqlite3_realloc(p->aiException, (n+p->nException)*sizeof(int));
    if( aNew ){
      int nNew = p->nException;
      const unsigned char *zCsr = (const unsigned char*)z;
      const unsigned char *zTerm = (const unsigned char*)&z[n];
      while( zCsr<zTerm ){
-        u32 iCode;
+        int iCode;
        int bToken;
        READ_UTF8(zCsr, zTerm, iCode);
        if( iCode<128 ){
          p->aTokenChar[iCode] = (unsigned char)bTokenChars;
        }else{
-          bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
+          bToken = sqlite3Fts5UnicodeIsalnum(iCode);
          assert( (bToken==0 || bToken==1) ); 
          assert( (bTokenChars==0 || bTokenChars==1) );
          if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
            int i;
            for(i=0; i<nNew; i++){
-              if( (u32)aNew[i]>iCode ) break;
+              if( aNew[i]>iCode ) break;
            }
            memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int));
            aNew[i] = iCode;
@@ -330,21 +322,6 @@ static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
  return;
 }

-static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
-  const char *z = zCat;
-
-  while( *z ){
-    while( *z==' ' || *z=='\t' ) z++;
-    if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
-      return SQLITE_ERROR;
-    }
-    while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
-  }
-
-  sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
-  return SQLITE_OK;
-}
-
 /*
 ** Create a "unicode61" tokenizer.
 */
@@ -363,54 +340,32 @@ static int fts5UnicodeCreate(
  }else{
    p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
    if( p ){
-      const char *zCat = "L* N* Co";
      int i;
      memset(p, 0, sizeof(Unicode61Tokenizer));
-
-      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
+      memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
+      p->bRemoveDiacritic = 1;
      p->nFold = 64;
-      p->aFold = sqlite3_malloc64(p->nFold * sizeof(char));
+      p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
      if( p->aFold==0 ){
        rc = SQLITE_NOMEM;
      }
-
-      /* Search for a "categories" argument */
-      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
-        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
-          zCat = azArg[i+1];
-        }
-      }
-
-      if( rc==SQLITE_OK ){
-        rc = unicodeSetCategories(p, zCat);
-      }
-
      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
        const char *zArg = azArg[i+1];
        if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
+          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
            rc = SQLITE_ERROR;
-          }else{
-            p->eRemoveDiacritic = (zArg[0] - '0');
-            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
-                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
-                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
-            );
          }
+          p->bRemoveDiacritic = (zArg[0]=='1');
        }else
        if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
          rc = fts5UnicodeAddExceptions(p, zArg, 1);
        }else
        if( 0==sqlite3_stricmp(azArg[i], "separators") ){
          rc = fts5UnicodeAddExceptions(p, zArg, 0);
-        }else
-        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
-          /* no-op */
        }else{
          rc = SQLITE_ERROR;
        }
      }
-
    }else{
      rc = SQLITE_NOMEM;
    }
@@ -429,10 +384,8 @@ static int fts5UnicodeCreate(
 ** character (not a separator).
 */
 static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
-  return (
-    p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)]
-    ^ fts5UnicodeIsException(p, iCode)
-  );
+  assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
+  return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode);
 }

 static int fts5UnicodeTokenize(
@@ -459,7 +412,7 @@ static int fts5UnicodeTokenize(
  /* Each iteration of this loop gobbles up a contiguous run of separators,
  ** then the next token.  */
  while( rc==SQLITE_OK ){
-    u32 iCode;                    /* non-ASCII codepoint read from input */
+    int iCode;                    /* non-ASCII codepoint read from input */
    char *zOut = aFold;
    int is;
    int ie;
@@ -491,7 +444,7 @@ static int fts5UnicodeTokenize(
      /* Grow the output buffer so that there is sufficient space to fit the
      ** largest possible utf-8 character.  */
      if( zOut>pEnd ){
-        aFold = sqlite3_malloc64((sqlite3_int64)nFold*2);
+        aFold = sqlite3_malloc(nFold*2);
        if( aFold==0 ){
          rc = SQLITE_NOMEM;
          goto tokenize_done;
@@ -510,7 +463,7 @@ static int fts5UnicodeTokenize(
        READ_UTF8(zCsr, zTerm, iCode);
        if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
 non_ascii_tokenchar:
-          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
+          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
          if( iCode ) WRITE_UTF8(zOut, iCode);
        }else{
          break;
@@ -1285,3 +1238,5 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){

  return rc;
 }
+
+
@@ -1,5 +1,5 @@
 /*
-** 2012-05-25
+** 2012 May 25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -18,6 +18,135 @@

 #include <assert.h>

+/*
+** Return true if the argument corresponds to a unicode codepoint
+** classified as either a letter or a number. Otherwise false.
+**
+** The results are undefined if the value passed to this function
+** is less than zero.
+*/
+int sqlite3Fts5UnicodeIsalnum(int c){
+  /* Each unsigned integer in the following array corresponds to a contiguous
+  ** range of unicode codepoints that are not either letters or numbers (i.e.
+  ** codepoints for which this function should return 0).
+  **
+  ** The most significant 22 bits in each 32-bit value contain the first 
+  ** codepoint in the range. The least significant 10 bits are used to store
+  ** the size of the range (always at least 1). In other words, the value 
+  ** ((C<<22) + N) represents a range of N codepoints starting with codepoint 
+  ** C. It is not possible to represent a range larger than 1023 codepoints 
+  ** using this format.
+  */
+  static const unsigned int aEntry[] = {
+    0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
+    0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
+    0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
+    0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
+    0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
+    0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
+    0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
+    0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
+    0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
+    0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
+    0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
+    0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
+    0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
+    0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
+    0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
+    0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
+    0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
+    0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
+    0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
+    0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
+    0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
+    0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
+    0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
+    0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
+    0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
+    0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
+    0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
+    0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
+    0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
+    0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
+    0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
+    0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
+    0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
+    0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
+    0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
+    0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
+    0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
+    0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
+    0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
+    0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
+    0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
+    0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
+    0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
+    0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
+    0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
+    0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
+    0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
+    0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
+    0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
+    0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
+    0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E,
+    0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100,
+    0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10,
+    0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402,
+    0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804,
+    0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012,
+    0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004,
+    0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002,
+    0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
+    0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
+    0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
+    0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
+    0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
+    0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
+    0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
+    0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
+    0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
+    0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
+    0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
+    0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
+    0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
+    0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
+    0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
+    0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
+    0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
+    0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
+    0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
+    0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
+    0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
+    0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
+    0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
+    0x380400F0,
+  };
+  static const unsigned int aAscii[4] = {
+    0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
+  };
+
+  if( (unsigned int)c<128 ){
+    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
+  }else if( (unsigned int)c<(1<<22) ){
+    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
+    int iRes = 0;
+    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
+    int iLo = 0;
+    while( iHi>=iLo ){
+      int iTest = (iHi + iLo) / 2;
+      if( key >= aEntry[iTest] ){
+        iRes = iTest;
+        iLo = iTest+1;
+      }else{
+        iHi = iTest-1;
+      }
+    }
+    assert( aEntry[0]<key );
+    assert( key>=aEntry[iRes] );
+    return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
+  }
+  return 1;
+}


 /*
@@ -28,48 +157,32 @@
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int fts5_remove_diacritic(int c, int bComplex){
+static int fts5_remove_diacritic(int c){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
-     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
-     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
-     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
-     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
-    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
-    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
-    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
-    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
-    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
-    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
-    63182, 63242, 63274, 63310, 63368, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
+     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
+     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
+     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
+    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
+    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
+    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
+    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
+    62924, 63050, 63082, 63274, 63390, 
  };
-#define HIBIT ((unsigned char)0x80)
-  unsigned char aChar[] = {
-    '\0',      'a',       'c',       'e',       'i',       'n',       
-    'o',       'u',       'y',       'y',       'a',       'c',       
-    'd',       'e',       'e',       'g',       'h',       'i',       
-    'j',       'k',       'l',       'n',       'o',       'r',       
-    's',       't',       'u',       'u',       'w',       'y',       
-    'z',       'o',       'u',       'a',       'i',       'o',       
-    'u',       'u'|HIBIT, 'a'|HIBIT, 'g',       'k',       'o',       
-    'o'|HIBIT, 'j',       'g',       'n',       'a'|HIBIT, 'a',       
-    'e',       'i',       'o',       'r',       'u',       's',       
-    't',       'h',       'a',       'e',       'o'|HIBIT, 'o',       
-    'o'|HIBIT, 'y',       '\0',      '\0',      '\0',      '\0',      
-    '\0',      '\0',      '\0',      '\0',      'a',       'b',       
-    'c'|HIBIT, 'd',       'd',       'e'|HIBIT, 'e',       'e'|HIBIT, 
-    'f',       'g',       'h',       'h',       'i',       'i'|HIBIT, 
-    'k',       'l',       'l'|HIBIT, 'l',       'm',       'n',       
-    'o'|HIBIT, 'p',       'r',       'r'|HIBIT, 'r',       's',       
-    's'|HIBIT, 't',       'u',       'u'|HIBIT, 'v',       'w',       
-    'w',       'x',       'y',       'z',       'h',       't',       
-    'w',       'y',       'a',       'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 
-    'e',       'e'|HIBIT, 'e'|HIBIT, 'i',       'o',       'o'|HIBIT, 
-    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',       
+  char aChar[] = {
+    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
+    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
+    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
+    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
+    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
+    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
+    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
+    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
+    'e',  'i',  'o',  'u',  'y',  
  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -86,8 +199,7 @@ static int fts5_remove_diacritic(int c, int bComplex){
    }
  }
  assert( key>=aDia[iRes] );
-  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
-  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
+  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
 }


@@ -100,8 +212,8 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){
  unsigned int mask1 = 0x000361F8;
  if( c<768 || c>817 ) return 0;
  return (c < 768+32) ?
-      (mask0 & ((unsigned int)1 << (c-768))) :
-      (mask1 & ((unsigned int)1 << (c-768-32)));
+      (mask0 & (1 << (c-768))) :
+      (mask1 & (1 << (c-768-32)));
 }


@@ -114,7 +226,7 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
+int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
@@ -237,9 +349,7 @@ int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
      assert( ret>0 );
    }

-    if( eRemoveDiacritic ){
-      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
-    }
+    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
  }
  
  else if( c>=66560 && c<66600 ){
@@ -248,529 +358,3 @@ int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){

  return ret;
 }
-
-
-int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
-  aArray[0] = 1;
-  switch( zCat[0] ){
-    case 'C':
-          switch( zCat[1] ){
-            case 'c': aArray[1] = 1; break;
-            case 'f': aArray[2] = 1; break;
-            case 'n': aArray[3] = 1; break;
-            case 's': aArray[4] = 1; break;
-            case 'o': aArray[31] = 1; break;
-            case '*': 
-              aArray[1] = 1;
-              aArray[2] = 1;
-              aArray[3] = 1;
-              aArray[4] = 1;
-              aArray[31] = 1;
-              break;
-            default: return 1;          }
-          break;
-
-    case 'L':
-          switch( zCat[1] ){
-            case 'l': aArray[5] = 1; break;
-            case 'm': aArray[6] = 1; break;
-            case 'o': aArray[7] = 1; break;
-            case 't': aArray[8] = 1; break;
-            case 'u': aArray[9] = 1; break;
-            case 'C': aArray[30] = 1; break;
-            case '*': 
-              aArray[5] = 1;
-              aArray[6] = 1;
-              aArray[7] = 1;
-              aArray[8] = 1;
-              aArray[9] = 1;
-              aArray[30] = 1;
-              break;
-            default: return 1;          }
-          break;
-
-    case 'M':
-          switch( zCat[1] ){
-            case 'c': aArray[10] = 1; break;
-            case 'e': aArray[11] = 1; break;
-            case 'n': aArray[12] = 1; break;
-            case '*': 
-              aArray[10] = 1;
-              aArray[11] = 1;
-              aArray[12] = 1;
-              break;
-            default: return 1;          }
-          break;
-
-    case 'N':
-          switch( zCat[1] ){
-            case 'd': aArray[13] = 1; break;
-            case 'l': aArray[14] = 1; break;
-            case 'o': aArray[15] = 1; break;
-            case '*': 
-              aArray[13] = 1;
-              aArray[14] = 1;
-              aArray[15] = 1;
-              break;
-            default: return 1;          }
-          break;
-
-    case 'P':
-          switch( zCat[1] ){
-            case 'c': aArray[16] = 1; break;
-            case 'd': aArray[17] = 1; break;
-            case 'e': aArray[18] = 1; break;
-            case 'f': aArray[19] = 1; break;
-            case 'i': aArray[20] = 1; break;
-            case 'o': aArray[21] = 1; break;
-            case 's': aArray[22] = 1; break;
-            case '*': 
-              aArray[16] = 1;
-              aArray[17] = 1;
-              aArray[18] = 1;
-              aArray[19] = 1;
-              aArray[20] = 1;
-              aArray[21] = 1;
-              aArray[22] = 1;
-              break;
-            default: return 1;          }
-          break;
-
-    case 'S':
-          switch( zCat[1] ){
-            case 'c': aArray[23] = 1; break;
-            case 'k': aArray[24] = 1; break;
-            case 'm': aArray[25] = 1; break;
-            case 'o': aArray[26] = 1; break;
-            case '*': 
-              aArray[23] = 1;
-              aArray[24] = 1;
-              aArray[25] = 1;
-              aArray[26] = 1;
-              break;
-            default: return 1;          }
-          break;
-
-    case 'Z':
-          switch( zCat[1] ){
-            case 'l': aArray[27] = 1; break;
-            case 'p': aArray[28] = 1; break;
-            case 's': aArray[29] = 1; break;
-            case '*': 
-              aArray[27] = 1;
-              aArray[28] = 1;
-              aArray[29] = 1;
-              break;
-            default: return 1;          }
-          break;
-
-  }
-  return 0;
-}
-
-static u16 aFts5UnicodeBlock[] = {
-    0,     1471,  1753,  1760,  1760,  1760,  1760,  1760,  1760,  1760,  
-    1760,  1760,  1760,  1760,  1760,  1763,  1765,  
-  };
-static u16 aFts5UnicodeMap[] = {
-    0,     32,    33,    36,    37,    40,    41,    42,    43,    44,    
-    45,    46,    48,    58,    60,    63,    65,    91,    92,    93,    
-    94,    95,    96,    97,    123,   124,   125,   126,   127,   160,   
-    161,   162,   166,   167,   168,   169,   170,   171,   172,   173,   
-    174,   175,   176,   177,   178,   180,   181,   182,   184,   185,   
-    186,   187,   188,   191,   192,   215,   216,   223,   247,   248,   
-    256,   312,   313,   329,   330,   377,   383,   385,   387,   388,   
-    391,   394,   396,   398,   402,   403,   405,   406,   409,   412,   
-    414,   415,   417,   418,   423,   427,   428,   431,   434,   436,   
-    437,   440,   442,   443,   444,   446,   448,   452,   453,   454,   
-    455,   456,   457,   458,   459,   460,   461,   477,   478,   496,   
-    497,   498,   499,   500,   503,   505,   506,   564,   570,   572,   
-    573,   575,   577,   580,   583,   584,   592,   660,   661,   688,   
-    706,   710,   722,   736,   741,   748,   749,   750,   751,   768,   
-    880,   884,   885,   886,   890,   891,   894,   900,   902,   903,   
-    904,   908,   910,   912,   913,   931,   940,   975,   977,   978,   
-    981,   984,   1008,  1012,  1014,  1015,  1018,  1020,  1021,  1072,  
-    1120,  1154,  1155,  1160,  1162,  1217,  1231,  1232,  1329,  1369,  
-    1370,  1377,  1417,  1418,  1423,  1425,  1470,  1471,  1472,  1473,  
-    1475,  1476,  1478,  1479,  1488,  1520,  1523,  1536,  1542,  1545,  
-    1547,  1548,  1550,  1552,  1563,  1566,  1568,  1600,  1601,  1611,  
-    1632,  1642,  1646,  1648,  1649,  1748,  1749,  1750,  1757,  1758,  
-    1759,  1765,  1767,  1769,  1770,  1774,  1776,  1786,  1789,  1791,  
-    1792,  1807,  1808,  1809,  1810,  1840,  1869,  1958,  1969,  1984,  
-    1994,  2027,  2036,  2038,  2039,  2042,  2048,  2070,  2074,  2075,  
-    2084,  2085,  2088,  2089,  2096,  2112,  2137,  2142,  2208,  2210,  
-    2276,  2304,  2307,  2308,  2362,  2363,  2364,  2365,  2366,  2369,  
-    2377,  2381,  2382,  2384,  2385,  2392,  2402,  2404,  2406,  2416,  
-    2417,  2418,  2425,  2433,  2434,  2437,  2447,  2451,  2474,  2482,  
-    2486,  2492,  2493,  2494,  2497,  2503,  2507,  2509,  2510,  2519,  
-    2524,  2527,  2530,  2534,  2544,  2546,  2548,  2554,  2555,  2561,  
-    2563,  2565,  2575,  2579,  2602,  2610,  2613,  2616,  2620,  2622,  
-    2625,  2631,  2635,  2641,  2649,  2654,  2662,  2672,  2674,  2677,  
-    2689,  2691,  2693,  2703,  2707,  2730,  2738,  2741,  2748,  2749,  
-    2750,  2753,  2759,  2761,  2763,  2765,  2768,  2784,  2786,  2790,  
-    2800,  2801,  2817,  2818,  2821,  2831,  2835,  2858,  2866,  2869,  
-    2876,  2877,  2878,  2879,  2880,  2881,  2887,  2891,  2893,  2902,  
-    2903,  2908,  2911,  2914,  2918,  2928,  2929,  2930,  2946,  2947,  
-    2949,  2958,  2962,  2969,  2972,  2974,  2979,  2984,  2990,  3006,  
-    3008,  3009,  3014,  3018,  3021,  3024,  3031,  3046,  3056,  3059,  
-    3065,  3066,  3073,  3077,  3086,  3090,  3114,  3125,  3133,  3134,  
-    3137,  3142,  3146,  3157,  3160,  3168,  3170,  3174,  3192,  3199,  
-    3202,  3205,  3214,  3218,  3242,  3253,  3260,  3261,  3262,  3263,  
-    3264,  3270,  3271,  3274,  3276,  3285,  3294,  3296,  3298,  3302,  
-    3313,  3330,  3333,  3342,  3346,  3389,  3390,  3393,  3398,  3402,  
-    3405,  3406,  3415,  3424,  3426,  3430,  3440,  3449,  3450,  3458,  
-    3461,  3482,  3507,  3517,  3520,  3530,  3535,  3538,  3542,  3544,  
-    3570,  3572,  3585,  3633,  3634,  3636,  3647,  3648,  3654,  3655,  
-    3663,  3664,  3674,  3713,  3716,  3719,  3722,  3725,  3732,  3737,  
-    3745,  3749,  3751,  3754,  3757,  3761,  3762,  3764,  3771,  3773,  
-    3776,  3782,  3784,  3792,  3804,  3840,  3841,  3844,  3859,  3860,  
-    3861,  3864,  3866,  3872,  3882,  3892,  3893,  3894,  3895,  3896,  
-    3897,  3898,  3899,  3900,  3901,  3902,  3904,  3913,  3953,  3967,  
-    3968,  3973,  3974,  3976,  3981,  3993,  4030,  4038,  4039,  4046,  
-    4048,  4053,  4057,  4096,  4139,  4141,  4145,  4146,  4152,  4153,  
-    4155,  4157,  4159,  4160,  4170,  4176,  4182,  4184,  4186,  4190,  
-    4193,  4194,  4197,  4199,  4206,  4209,  4213,  4226,  4227,  4229,  
-    4231,  4237,  4238,  4239,  4240,  4250,  4253,  4254,  4256,  4295,  
-    4301,  4304,  4347,  4348,  4349,  4682,  4688,  4696,  4698,  4704,  
-    4746,  4752,  4786,  4792,  4800,  4802,  4808,  4824,  4882,  4888,  
-    4957,  4960,  4969,  4992,  5008,  5024,  5120,  5121,  5741,  5743,  
-    5760,  5761,  5787,  5788,  5792,  5867,  5870,  5888,  5902,  5906,  
-    5920,  5938,  5941,  5952,  5970,  5984,  5998,  6002,  6016,  6068,  
-    6070,  6071,  6078,  6086,  6087,  6089,  6100,  6103,  6104,  6107,  
-    6108,  6109,  6112,  6128,  6144,  6150,  6151,  6155,  6158,  6160,  
-    6176,  6211,  6212,  6272,  6313,  6314,  6320,  6400,  6432,  6435,  
-    6439,  6441,  6448,  6450,  6451,  6457,  6464,  6468,  6470,  6480,  
-    6512,  6528,  6576,  6593,  6600,  6608,  6618,  6622,  6656,  6679,  
-    6681,  6686,  6688,  6741,  6742,  6743,  6744,  6752,  6753,  6754,  
-    6755,  6757,  6765,  6771,  6783,  6784,  6800,  6816,  6823,  6824,  
-    6912,  6916,  6917,  6964,  6965,  6966,  6971,  6972,  6973,  6978,  
-    6979,  6981,  6992,  7002,  7009,  7019,  7028,  7040,  7042,  7043,  
-    7073,  7074,  7078,  7080,  7082,  7083,  7084,  7086,  7088,  7098,  
-    7142,  7143,  7144,  7146,  7149,  7150,  7151,  7154,  7164,  7168,  
-    7204,  7212,  7220,  7222,  7227,  7232,  7245,  7248,  7258,  7288,  
-    7294,  7360,  7376,  7379,  7380,  7393,  7394,  7401,  7405,  7406,  
-    7410,  7412,  7413,  7424,  7468,  7531,  7544,  7545,  7579,  7616,  
-    7676,  7680,  7830,  7838,  7936,  7944,  7952,  7960,  7968,  7976,  
-    7984,  7992,  8000,  8008,  8016,  8025,  8027,  8029,  8031,  8033,  
-    8040,  8048,  8064,  8072,  8080,  8088,  8096,  8104,  8112,  8118,  
-    8120,  8124,  8125,  8126,  8127,  8130,  8134,  8136,  8140,  8141,  
-    8144,  8150,  8152,  8157,  8160,  8168,  8173,  8178,  8182,  8184,  
-    8188,  8189,  8192,  8203,  8208,  8214,  8216,  8217,  8218,  8219,  
-    8221,  8222,  8223,  8224,  8232,  8233,  8234,  8239,  8240,  8249,  
-    8250,  8251,  8255,  8257,  8260,  8261,  8262,  8263,  8274,  8275,  
-    8276,  8277,  8287,  8288,  8298,  8304,  8305,  8308,  8314,  8317,  
-    8318,  8319,  8320,  8330,  8333,  8334,  8336,  8352,  8400,  8413,  
-    8417,  8418,  8421,  8448,  8450,  8451,  8455,  8456,  8458,  8459,  
-    8462,  8464,  8467,  8468,  8469,  8470,  8472,  8473,  8478,  8484,  
-    8485,  8486,  8487,  8488,  8489,  8490,  8494,  8495,  8496,  8500,  
-    8501,  8505,  8506,  8508,  8510,  8512,  8517,  8519,  8522,  8523,  
-    8524,  8526,  8527,  8528,  8544,  8579,  8581,  8585,  8592,  8597,  
-    8602,  8604,  8608,  8609,  8611,  8612,  8614,  8615,  8622,  8623,  
-    8654,  8656,  8658,  8659,  8660,  8661,  8692,  8960,  8968,  8972,  
-    8992,  8994,  9001,  9002,  9003,  9084,  9085,  9115,  9140,  9180,  
-    9186,  9216,  9280,  9312,  9372,  9450,  9472,  9655,  9656,  9665,  
-    9666,  9720,  9728,  9839,  9840,  9985,  10088, 10089, 10090, 10091, 
-    10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, 
-    10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217, 
-    10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627, 
-    10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637, 
-    10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, 
-    10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750, 
-    11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365, 
-    11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393, 
-    11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520, 
-    11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696, 
-    11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780, 
-    11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800, 
-    11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812, 
-    11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904, 
-    11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296, 
-    12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306, 
-    12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317, 
-    12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347, 
-    12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449, 
-    12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736, 
-    12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938, 
-    12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981, 
-    40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528, 
-    42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624, 
-    42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800, 
-    42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912, 
-    43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043, 
-    43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136, 
-    43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264, 
-    43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395, 
-    43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472, 
-    43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588, 
-    43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643, 
-    43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713, 
-    43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762, 
-    43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003, 
-    44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203, 
-    55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112, 
-    64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320, 
-    64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020, 
-    65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075, 
-    65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086, 
-    65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097, 
-    65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118, 
-    65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279, 
-    65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294, 
-    65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343, 
-    65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378, 
-    65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490, 
-    65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529, 
-    65532, 0,     13,    40,    60,    63,    80,    128,   256,   263,   
-    311,   320,   373,   377,   394,   400,   464,   509,   640,   672,   
-    768,   800,   816,   833,   834,   842,   896,   927,   928,   968,   
-    976,   977,   1024,  1064,  1104,  1184,  2048,  2056,  2058,  2103,  
-    2108,  2111,  2135,  2136,  2304,  2326,  2335,  2336,  2367,  2432,  
-    2494,  2560,  2561,  2565,  2572,  2576,  2581,  2585,  2616,  2623,  
-    2624,  2640,  2656,  2685,  2687,  2816,  2873,  2880,  2904,  2912,  
-    2936,  3072,  3680,  4096,  4097,  4098,  4099,  4152,  4167,  4178,  
-    4198,  4224,  4226,  4227,  4272,  4275,  4279,  4281,  4283,  4285,  
-    4286,  4304,  4336,  4352,  4355,  4391,  4396,  4397,  4406,  4416,  
-    4480,  4482,  4483,  4531,  4534,  4543,  4545,  4549,  4560,  5760,  
-    5803,  5804,  5805,  5806,  5808,  5814,  5815,  5824,  8192,  9216,  
-    9328,  12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248, 
-    53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637, 
-    53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298, 
-    54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441, 
-    54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541, 
-    54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662, 
-    54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922, 
-    54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062, 
-    55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178, 
-    55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961, 
-    60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003, 
-    61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028, 
-    61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099, 
-    61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744, 
-    61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368, 
-    62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971, 
-    63045, 63104, 63232, 0,     42710, 42752, 46900, 46912, 47133, 63488, 
-    1,     32,    256,   0,     65533, 
-  };
-static u16 aFts5UnicodeData[] = {
-    1025,  61,    117,   55,    117,   54,    50,    53,    57,    53,    
-    49,    85,    333,   85,    121,   85,    841,   54,    53,    50,    
-    56,    48,    56,    837,   54,    57,    50,    57,    1057,  61,    
-    53,    151,   58,    53,    56,    58,    39,    52,    57,    34,    
-    58,    56,    58,    57,    79,    56,    37,    85,    56,    47,    
-    39,    51,    111,   53,    745,   57,    233,   773,   57,    261,   
-    1822,  37,    542,   37,    1534,  222,   69,    73,    37,    126,   
-    126,   73,    69,    137,   37,    73,    37,    105,   101,   73,    
-    37,    73,    37,    190,   158,   37,    126,   126,   73,    37,    
-    126,   94,    37,    39,    94,    69,    135,   41,    40,    37,    
-    41,    40,    37,    41,    40,    37,    542,   37,    606,   37,    
-    41,    40,    37,    126,   73,    37,    1886,  197,   73,    37,    
-    73,    69,    126,   105,   37,    286,   2181,  39,    869,   582,   
-    152,   390,   472,   166,   248,   38,    56,    38,    568,   3596,  
-    158,   38,    56,    94,    38,    101,   53,    88,    41,    53,    
-    105,   41,    73,    37,    553,   297,   1125,  94,    37,    105,   
-    101,   798,   133,   94,    57,    126,   94,    37,    1641,  1541,  
-    1118,  58,    172,   75,    1790,  478,   37,    2846,  1225,  38,    
-    213,   1253,  53,    49,    55,    1452,  49,    44,    53,    76,    
-    53,    76,    53,    44,    871,   103,   85,    162,   121,   85,    
-    55,    85,    90,    364,   53,    85,    1031,  38,    327,   684,   
-    333,   149,   71,    44,    3175,  53,    39,    236,   34,    58,    
-    204,   70,    76,    58,    140,   71,    333,   103,   90,    39,    
-    469,   34,    39,    44,    967,   876,   2855,  364,   39,    333,   
-    1063,  300,   70,    58,    117,   38,    711,   140,   38,    300,   
-    38,    108,   38,    172,   501,   807,   108,   53,    39,    359,   
-    876,   108,   42,    1735,  44,    42,    44,    39,    106,   268,   
-    138,   44,    74,    39,    236,   327,   76,    85,    333,   53,    
-    38,    199,   231,   44,    74,    263,   71,    711,   231,   39,    
-    135,   44,    39,    106,   140,   74,    74,    44,    39,    42,    
-    71,    103,   76,    333,   71,    87,    207,   58,    55,    76,    
-    42,    199,   71,    711,   231,   71,    71,    71,    44,    106,   
-    76,    76,    108,   44,    135,   39,    333,   76,    103,   44,    
-    76,    42,    295,   103,   711,   231,   71,    167,   44,    39,    
-    106,   172,   76,    42,    74,    44,    39,    71,    76,    333,   
-    53,    55,    44,    74,    263,   71,    711,   231,   71,    167,   
-    44,    39,    42,    44,    42,    140,   74,    74,    44,    44,    
-    42,    71,    103,   76,    333,   58,    39,    207,   44,    39,    
-    199,   103,   135,   71,    39,    71,    71,    103,   391,   74,    
-    44,    74,    106,   106,   44,    39,    42,    333,   111,   218,   
-    55,    58,    106,   263,   103,   743,   327,   167,   39,    108,   
-    138,   108,   140,   76,    71,    71,    76,    333,   239,   58,    
-    74,    263,   103,   743,   327,   167,   44,    39,    42,    44,    
-    170,   44,    74,    74,    76,    74,    39,    71,    76,    333,   
-    71,    74,    263,   103,   1319,  39,    106,   140,   106,   106,   
-    44,    39,    42,    71,    76,    333,   207,   58,    199,   74,    
-    583,   775,   295,   39,    231,   44,    106,   108,   44,    266,   
-    74,    53,    1543,  44,    71,    236,   55,    199,   38,    268,   
-    53,    333,   85,    71,    39,    71,    39,    39,    135,   231,   
-    103,   39,    39,    71,    135,   44,    71,    204,   76,    39,    
-    167,   38,    204,   333,   135,   39,    122,   501,   58,    53,    
-    122,   76,    218,   333,   335,   58,    44,    58,    44,    58,    
-    44,    54,    50,    54,    50,    74,    263,   1159,  460,   42,    
-    172,   53,    76,    167,   364,   1164,  282,   44,    218,   90,    
-    181,   154,   85,    1383,  74,    140,   42,    204,   42,    76,    
-    74,    76,    39,    333,   213,   199,   74,    76,    135,   108,   
-    39,    106,   71,    234,   103,   140,   423,   44,    74,    76,    
-    202,   44,    39,    42,    333,   106,   44,    90,    1225,  41,    
-    41,    1383,  53,    38,    10631, 135,   231,   39,    135,   1319,  
-    135,   1063,  135,   231,   39,    135,   487,   1831,  135,   2151,  
-    108,   309,   655,   519,   346,   2727,  49,    19847, 85,    551,   
-    61,    839,   54,    50,    2407,  117,   110,   423,   135,   108,   
-    583,   108,   85,    583,   76,    423,   103,   76,    1671,  76,    
-    42,    236,   266,   44,    74,    364,   117,   38,    117,   55,    
-    39,    44,    333,   335,   213,   49,    149,   108,   61,    333,   
-    1127,  38,    1671,  1319,  44,    39,    2247,  935,   108,   138,   
-    76,    106,   74,    44,    202,   108,   58,    85,    333,   967,   
-    167,   1415,  554,   231,   74,    333,   47,    1114,  743,   76,    
-    106,   85,    1703,  42,    44,    42,    236,   44,    42,    44,    
-    74,    268,   202,   332,   44,    333,   333,   245,   38,    213,   
-    140,   42,    1511,  44,    42,    172,   42,    44,    170,   44,    
-    74,    231,   333,   245,   346,   300,   314,   76,    42,    967,   
-    42,    140,   74,    76,    42,    44,    74,    71,    333,   1415,  
-    44,    42,    76,    106,   44,    42,    108,   74,    149,   1159,  
-    266,   268,   74,    76,    181,   333,   103,   333,   967,   198,   
-    85,    277,   108,   53,    428,   42,    236,   135,   44,    135,   
-    74,    44,    71,    1413,  2022,  421,   38,    1093,  1190,  1260,  
-    140,   4830,  261,   3166,  261,   265,   197,   201,   261,   265,   
-    261,   265,   197,   201,   261,   41,    41,    41,    94,    229,   
-    265,   453,   261,   264,   261,   264,   261,   264,   165,   69,    
-    137,   40,    56,    37,    120,   101,   69,    137,   40,    120,   
-    133,   69,    137,   120,   261,   169,   120,   101,   69,    137,   
-    40,    88,    381,   162,   209,   85,    52,    51,    54,    84,    
-    51,    54,    52,    277,   59,    60,    162,   61,    309,   52,    
-    51,    149,   80,    117,   57,    54,    50,    373,   57,    53,    
-    48,    341,   61,    162,   194,   47,    38,    207,   121,   54,    
-    50,    38,    335,   121,   54,    50,    422,   855,   428,   139,   
-    44,    107,   396,   90,    41,    154,   41,    90,    37,    105,   
-    69,    105,   37,    58,    41,    90,    57,    169,   218,   41,    
-    58,    41,    58,    41,    58,    137,   58,    37,    137,   37,    
-    135,   37,    90,    69,    73,    185,   94,    101,   58,    57,    
-    90,    37,    58,    527,   1134,  94,    142,   47,    185,   186,   
-    89,    154,   57,    90,    57,    90,    57,    250,   57,    1018,  
-    89,    90,    57,    58,    57,    1018,  8601,  282,   153,   666,   
-    89,    250,   54,    50,    2618,  57,    986,   825,   1306,  217,   
-    602,   1274,  378,   1935,  2522,  719,   5882,  57,    314,   57,    
-    1754,  281,   3578,  57,    4634,  3322,  54,    50,    54,    50,    
-    54,    50,    54,    50,    54,    50,    54,    50,    54,    50,    
-    975,   1434,  185,   54,    50,    1017,  54,    50,    54,    50,    
-    54,    50,    54,    50,    54,    50,    537,   8218,  4217,  54,    
-    50,    54,    50,    54,    50,    54,    50,    54,    50,    54,    
-    50,    54,    50,    54,    50,    54,    50,    54,    50,    54,    
-    50,    2041,  54,    50,    54,    50,    1049,  54,    50,    8281,  
-    1562,  697,   90,    217,   346,   1513,  1509,  126,   73,    69,    
-    254,   105,   37,    94,    37,    94,    165,   70,    105,   37,    
-    3166,  37,    218,   158,   108,   94,    149,   47,    85,    1221,  
-    37,    37,    1799,  38,    53,    44,    743,   231,   231,   231,   
-    231,   231,   231,   231,   231,   1036,  85,    52,    51,    52,    
-    51,    117,   52,    51,    53,    52,    51,    309,   49,    85,    
-    49,    53,    52,    51,    85,    52,    51,    54,    50,    54,    
-    50,    54,    50,    54,    50,    181,   38,    341,   81,    858,   
-    2874,  6874,  410,   61,    117,   58,    38,    39,    46,    54,    
-    50,    54,    50,    54,    50,    54,    50,    54,    50,    90,    
-    54,    50,    54,    50,    54,    50,    54,    50,    49,    54,    
-    82,    58,    302,   140,   74,    49,    166,   90,    110,   38,    
-    39,    53,    90,    2759,  76,    88,    70,    39,    49,    2887,  
-    53,    102,   39,    1319,  3015,  90,    143,   346,   871,   1178,  
-    519,   1018,  335,   986,   271,   58,    495,   1050,  335,   1274,  
-    495,   2042,  8218,  39,    39,    2074,  39,    39,    679,   38,    
-    36583, 1786,  1287,  198,   85,    8583,  38,    117,   519,   333,   
-    71,    1502,  39,    44,    107,   53,    332,   53,    38,    798,   
-    44,    2247,  334,   76,    213,   760,   294,   88,    478,   69,    
-    2014,  38,    261,   190,   350,   38,    88,    158,   158,   382,   
-    70,    37,    231,   44,    103,   44,    135,   44,    743,   74,    
-    76,    42,    154,   207,   90,    55,    58,    1671,  149,   74,    
-    1607,  522,   44,    85,    333,   588,   199,   117,   39,    333,   
-    903,   268,   85,    743,   364,   74,    53,    935,   108,   42,    
-    1511,  44,    74,    140,   74,    44,    138,   437,   38,    333,   
-    85,    1319,  204,   74,    76,    74,    76,    103,   44,    263,   
-    44,    42,    333,   149,   519,   38,    199,   122,   39,    42,    
-    1543,  44,    39,    108,   71,    76,    167,   76,    39,    44,    
-    39,    71,    38,    85,    359,   42,    76,    74,    85,    39,    
-    70,    42,    44,    199,   199,   199,   231,   231,   1127,  74,    
-    44,    74,    44,    74,    53,    42,    44,    333,   39,    39,    
-    743,   1575,  36,    68,    68,    36,    63,    63,    11719, 3399,  
-    229,   165,   39,    44,    327,   57,    423,   167,   39,    71,    
-    71,    3463,  536,   11623, 54,    50,    2055,  1735,  391,   55,    
-    58,    524,   245,   54,    50,    53,    236,   53,    81,    80,    
-    54,    50,    54,    50,    54,    50,    54,    50,    54,    50,    
-    54,    50,    54,    50,    54,    50,    85,    54,    50,    149,   
-    112,   117,   149,   49,    54,    50,    54,    50,    54,    50,    
-    117,   57,    49,    121,   53,    55,    85,    167,   4327,  34,    
-    117,   55,    117,   54,    50,    53,    57,    53,    49,    85,    
-    333,   85,    121,   85,    841,   54,    53,    50,    56,    48,    
-    56,    837,   54,    57,    50,    57,    54,    50,    53,    54,    
-    50,    85,    327,   38,    1447,  70,    999,   199,   199,   199,   
-    103,   87,    57,    56,    58,    87,    58,    153,   90,    98,    
-    90,    391,   839,   615,   71,    487,   455,   3943,  117,   1455,  
-    314,   1710,  143,   570,   47,    410,   1466,  44,    935,   1575,  
-    999,   143,   551,   46,    263,   46,    967,   53,    1159,  263,   
-    53,    174,   1289,  1285,  2503,  333,   199,   39,    1415,  71,    
-    39,    743,   53,    271,   711,   207,   53,    839,   53,    1799,  
-    71,    39,    108,   76,    140,   135,   103,   871,   108,   44,    
-    271,   309,   935,   79,    53,    1735,  245,   711,   271,   615,   
-    271,   2343,  1007,  42,    44,    42,    1703,  492,   245,   655,   
-    333,   76,    42,    1447,  106,   140,   74,    76,    85,    34,    
-    149,   807,   333,   108,   1159,  172,   42,    268,   333,   149,   
-    76,    42,    1543,  106,   300,   74,    135,   149,   333,   1383,  
-    44,    42,    44,    74,    204,   42,    44,    333,   28135, 3182,  
-    149,   34279, 18215, 2215,  39,    1482,  140,   422,   71,    7898,  
-    1274,  1946,  74,    108,   122,   202,   258,   268,   90,    236,   
-    986,   140,   1562,  2138,  108,   58,    2810,  591,   841,   837,   
-    841,   229,   581,   841,   837,   41,    73,    41,    73,    137,   
-    265,   133,   37,    229,   357,   841,   837,   73,    137,   265,   
-    233,   837,   73,    137,   169,   41,    233,   837,   841,   837,   
-    841,   837,   841,   837,   841,   837,   841,   837,   841,   901,   
-    809,   57,    805,   57,    197,   809,   57,    805,   57,    197,   
-    809,   57,    805,   57,    197,   809,   57,    805,   57,    197,   
-    809,   57,    805,   57,    197,   94,    1613,  135,   871,   71,    
-    39,    39,    327,   135,   39,    39,    39,    39,    39,    39,    
-    103,   71,    39,    39,    39,    39,    39,    39,    71,    39,    
-    135,   231,   135,   135,   39,    327,   551,   103,   167,   551,   
-    89,    1434,  3226,  506,   474,   506,   506,   367,   1018,  1946,  
-    1402,  954,   1402,  314,   90,    1082,  218,   2266,  666,   1210,  
-    186,   570,   2042,  58,    5850,  154,   2010,  154,   794,   2266,  
-    378,   2266,  3738,  39,    39,    39,    39,    39,    39,    17351, 
-    34,    3074,  7692,  63,    63,    
-  };
-
-int sqlite3Fts5UnicodeCategory(u32 iCode) { 
-  int iRes = -1;
-  int iHi;
-  int iLo;
-  int ret;
-  u16 iKey;
-
-  if( iCode>=(1<<20) ){
-    return 0;
-  }
-  iLo = aFts5UnicodeBlock[(iCode>>16)];
-  iHi = aFts5UnicodeBlock[1+(iCode>>16)];
-  iKey = (iCode & 0xFFFF);
-  while( iHi>iLo ){
-    int iTest = (iHi + iLo) / 2;
-    assert( iTest>=iLo && iTest<iHi );
-    if( iKey>=aFts5UnicodeMap[iTest] ){
-      iRes = iTest;
-      iLo = iTest+1;
-    }else{
-      iHi = iTest;
-    }
-  }
-
-  if( iRes<0 ) return 0;
-  if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
-  ret = aFts5UnicodeData[iRes] & 0x1F;
-  if( ret!=30 ) return ret;
-  return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9;
-}
-
-void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
-  int i = 0;
-  int iTbl = 0;
-  while( i<128 ){
-    int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
-    int n = (aFts5UnicodeData[iTbl] >> 5) + i;
-    for(; i<128 && i<n; i++){
-      aAscii[i] = (u8)bToken;
-    }
-    iTbl++;
-  }
-}
@@ -76,7 +76,7 @@ int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){
    u8 n;
    p -= 2;
    n = sqlite3Fts5GetVarint(p, &v64);
-    *v = ((u32)v64) & 0x7FFFFFFF;
+    *v = (u32)v64;
    assert( n>3 && n<=9 );
    return n;
  }
@@ -342,3 +342,4 @@ int sqlite3Fts5GetVarintLen(u32 iVal){
  if( iVal<(1 << 28) ) return 4;
  return 5;
 }
+
@@ -29,11 +29,6 @@
 **   the number of fts5 rows that contain at least one instance of term
 **   $term. Field $cnt is set to the total number of instances of term 
 **   $term in the database.
-**
-** instance:
-**     CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY(<all-fields>));
-**
-**   One row for each term instance in the database. 
 */


@@ -49,14 +44,13 @@ struct Fts5VocabTable {
  char *zFts5Db;                  /* Db containing fts5 table */
  sqlite3 *db;                    /* Database handle */
  Fts5Global *pGlobal;            /* FTS5 global object for this database */
-  int eType;                      /* FTS5_VOCAB_COL, ROW or INSTANCE */
-  unsigned bBusy;                 /* True if busy */
+  int eType;                      /* FTS5_VOCAB_COL or ROW */
 };

 struct Fts5VocabCursor {
  sqlite3_vtab_cursor base;
  sqlite3_stmt *pStmt;            /* Statement holding lock on pIndex */
-  Fts5Table *pFts5;               /* Associated FTS5 table */
+  Fts5Index *pIndex;              /* Associated FTS5 index */

  int bEof;                       /* True if this cursor is at EOF */
  Fts5IndexIter *pIter;           /* Term/rowid iterator object */
@@ -65,26 +59,21 @@ struct Fts5VocabCursor {
  char *zLeTerm;                  /* (term <= $zLeTerm) paramater, or NULL */

  /* These are used by 'col' tables only */
+  Fts5Config *pConfig;            /* Fts5 table configuration */
  int iCol;
  i64 *aCnt;
  i64 *aDoc;

-  /* Output values used by all tables. */
+  /* Output values used by 'row' and 'col' tables */
  i64 rowid;                      /* This table's current rowid value */
  Fts5Buffer term;                /* Current value of 'term' column */
-
-  /* Output values Used by 'instance' tables only */
-  i64 iInstPos;
-  int iInstOff;
 };

-#define FTS5_VOCAB_COL      0
-#define FTS5_VOCAB_ROW      1
-#define FTS5_VOCAB_INSTANCE 2
+#define FTS5_VOCAB_COL    0
+#define FTS5_VOCAB_ROW    1

 #define FTS5_VOCAB_COL_SCHEMA  "term, col, doc, cnt"
 #define FTS5_VOCAB_ROW_SCHEMA  "term, doc, cnt"
-#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset"

 /*
 ** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
@@ -112,9 +101,6 @@ static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
    if( sqlite3_stricmp(zCopy, "row")==0 ){
      *peType = FTS5_VOCAB_ROW;
    }else
-    if( sqlite3_stricmp(zCopy, "instance")==0 ){
-      *peType = FTS5_VOCAB_INSTANCE;
-    }else
    {
      *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy);
      rc = SQLITE_ERROR;
@@ -175,8 +161,7 @@ static int fts5VocabInitVtab(
 ){
  const char *azSchema[] = { 
    "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA  ")", 
-    "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA  ")",
-    "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")"
+    "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA  ")"
  };

  Fts5VocabTable *pRet = 0;
@@ -250,15 +235,6 @@ static int fts5VocabCreateMethod(

 /* 
 ** Implementation of the xBestIndex method.
-**
-** Only constraints of the form:
-**
-**     term <= ?
-**     term == ?
-**     term >= ?
-**
-** are interpreted. Less-than and less-than-or-equal are treated 
-** identically, as are greater-than and greater-than-or-equal.
 */
 static int fts5VocabBestIndexMethod(
  sqlite3_vtab *pUnused,
@@ -327,18 +303,13 @@ static int fts5VocabOpenMethod(
  sqlite3_vtab_cursor **ppCsr
 ){
  Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
-  Fts5Table *pFts5 = 0;
+  Fts5Index *pIndex = 0;
+  Fts5Config *pConfig = 0;
  Fts5VocabCursor *pCsr = 0;
  int rc = SQLITE_OK;
  sqlite3_stmt *pStmt = 0;
  char *zSql = 0;

-  if( pTab->bBusy ){
-    pVTab->zErrMsg = sqlite3_mprintf(
-       "recursive definition for %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
-    );
-    return SQLITE_ERROR;
-  }
  zSql = sqlite3Fts5Mprintf(&rc,
      "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
      pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
@@ -350,38 +321,33 @@ static int fts5VocabOpenMethod(
  assert( rc==SQLITE_OK || pStmt==0 );
  if( rc==SQLITE_ERROR ) rc = SQLITE_OK;

-  pTab->bBusy = 1;
  if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
    i64 iId = sqlite3_column_int64(pStmt, 0);
-    pFts5 = sqlite3Fts5TableFromCsrid(pTab->pGlobal, iId);
+    pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig);
  }
-  pTab->bBusy = 0;

-  if( rc==SQLITE_OK ){
-    if( pFts5==0 ){
-      rc = sqlite3_finalize(pStmt);
-      pStmt = 0;
-      if( rc==SQLITE_OK ){
-        pVTab->zErrMsg = sqlite3_mprintf(
-            "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
-            );
-        rc = SQLITE_ERROR;
-      }
-    }else{
-      rc = sqlite3Fts5FlushToDisk(pFts5);
+  if( rc==SQLITE_OK && pIndex==0 ){
+    rc = sqlite3_finalize(pStmt);
+    pStmt = 0;
+    if( rc==SQLITE_OK ){
+      pVTab->zErrMsg = sqlite3_mprintf(
+          "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
+      );
+      rc = SQLITE_ERROR;
    }
  }

  if( rc==SQLITE_OK ){
-    int nByte = pFts5->pConfig->nCol * sizeof(i64)*2 + sizeof(Fts5VocabCursor);
+    int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
    pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
  }

  if( pCsr ){
-    pCsr->pFts5 = pFts5;
+    pCsr->pIndex = pIndex;
    pCsr->pStmt = pStmt;
+    pCsr->pConfig = pConfig;
    pCsr->aCnt = (i64*)&pCsr[1];
-    pCsr->aDoc = &pCsr->aCnt[pFts5->pConfig->nCol];
+    pCsr->aDoc = &pCsr->aCnt[pConfig->nCol];
  }else{
    sqlite3_finalize(pStmt);
  }
@@ -397,7 +363,6 @@ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
  sqlite3_free(pCsr->zLeTerm);
  pCsr->nLeTerm = -1;
  pCsr->zLeTerm = 0;
-  pCsr->bEof = 0;
 }

 /*
@@ -413,56 +378,6 @@ static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){
  return SQLITE_OK;
 }

-static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){
-  int rc = SQLITE_OK;
-  
-  if( sqlite3Fts5IterEof(pCsr->pIter) ){
-    pCsr->bEof = 1;
-  }else{
-    const char *zTerm;
-    int nTerm;
-    zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
-    if( pCsr->nLeTerm>=0 ){
-      int nCmp = MIN(nTerm, pCsr->nLeTerm);
-      int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
-      if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
-        pCsr->bEof = 1;
-      }
-    }
-
-    sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
-  }
-  return rc;
-}
-
-static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
-  int eDetail = pCsr->pFts5->pConfig->eDetail;
-  int rc = SQLITE_OK;
-  Fts5IndexIter *pIter = pCsr->pIter;
-  i64 *pp = &pCsr->iInstPos;
-  int *po = &pCsr->iInstOff;
-  
-  assert( sqlite3Fts5IterEof(pIter)==0 );
-  assert( pCsr->bEof==0 );
-  while( eDetail==FTS5_DETAIL_NONE
-      || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) 
-  ){
-    pCsr->iInstPos = 0;
-    pCsr->iInstOff = 0;
-
-    rc = sqlite3Fts5IterNextScan(pCsr->pIter);
-    if( rc==SQLITE_OK ){
-      rc = fts5VocabInstanceNewTerm(pCsr);
-      if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE ) break;
-    }
-    if( rc ){
-      pCsr->bEof = 1;
-      break;
-    }
-  }
-
-  return rc;
-}

 /*
 ** Advance the cursor to the next row in the table.
@@ -471,21 +386,17 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
  Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
  int rc = SQLITE_OK;
-  int nCol = pCsr->pFts5->pConfig->nCol;
+  int nCol = pCsr->pConfig->nCol;

  pCsr->rowid++;

-  if( pTab->eType==FTS5_VOCAB_INSTANCE ){
-    return fts5VocabInstanceNext(pCsr);
-  }
-
  if( pTab->eType==FTS5_VOCAB_COL ){
    for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
      if( pCsr->aDoc[pCsr->iCol] ) break;
    }
  }

-  if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){
+  if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){
    if( sqlite3Fts5IterEof(pCsr->pIter) ){
      pCsr->bEof = 1;
    }else{
@@ -493,7 +404,6 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
      int nTerm;

      zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
-      assert( nTerm>=0 );
      if( pCsr->nLeTerm>=0 ){
        int nCmp = MIN(nTerm, pCsr->nLeTerm);
        int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
@@ -510,29 +420,26 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){

      assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
      while( rc==SQLITE_OK ){
-        int eDetail = pCsr->pFts5->pConfig->eDetail;
        const u8 *pPos; int nPos;   /* Position list */
        i64 iPos = 0;               /* 64-bit position read from poslist */
        int iOff = 0;               /* Current offset within position list */

        pPos = pCsr->pIter->pData;
        nPos = pCsr->pIter->nData;
-
-        switch( pTab->eType ){
-          case FTS5_VOCAB_ROW:
-            if( eDetail==FTS5_DETAIL_FULL ){
+        switch( pCsr->pConfig->eDetail ){
+          case FTS5_DETAIL_FULL:
+            pPos = pCsr->pIter->pData;
+            nPos = pCsr->pIter->nData;
+            if( pTab->eType==FTS5_VOCAB_ROW ){
              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
                pCsr->aCnt[0]++;
              }
-            }
-            pCsr->aDoc[0]++;
-            break;
-
-          case FTS5_VOCAB_COL:
-            if( eDetail==FTS5_DETAIL_FULL ){
+              pCsr->aDoc[0]++;
+            }else{
              int iCol = -1;
              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
                int ii = FTS5_POS2COLUMN(iPos);
+                pCsr->aCnt[ii]++;
                if( iCol!=ii ){
                  if( ii>=nCol ){
                    rc = FTS5_CORRUPT;
@@ -541,9 +448,14 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
                  pCsr->aDoc[ii]++;
                  iCol = ii;
                }
-                pCsr->aCnt[ii]++;
              }
-            }else if( eDetail==FTS5_DETAIL_COLUMNS ){
+            }
+            break;
+
+          case FTS5_DETAIL_COLUMNS:
+            if( pTab->eType==FTS5_VOCAB_ROW ){
+              pCsr->aDoc[0]++;
+            }else{
              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
                assert_nc( iPos>=0 && iPos<nCol );
                if( iPos>=nCol ){
@@ -552,27 +464,22 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
                }
                pCsr->aDoc[iPos]++;
              }
-            }else{
-              assert( eDetail==FTS5_DETAIL_NONE );
-              pCsr->aDoc[0]++;
            }
            break;

-          default:
-            assert( pTab->eType==FTS5_VOCAB_INSTANCE );
+          default: 
+            assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE );
+            pCsr->aDoc[0]++;
            break;
        }

        if( rc==SQLITE_OK ){
          rc = sqlite3Fts5IterNextScan(pCsr->pIter);
        }
-        if( pTab->eType==FTS5_VOCAB_INSTANCE ) break;

        if( rc==SQLITE_OK ){
          zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
-          if( nTerm!=pCsr->term.n 
-          || (nTerm>0 && memcmp(zTerm, pCsr->term.p, nTerm)) 
-          ){
+          if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){
            break;
          }
          if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
@@ -582,10 +489,8 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
  }

  if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
-    for(/* noop */; pCsr->iCol<nCol && pCsr->aDoc[pCsr->iCol]==0; pCsr->iCol++);
-    if( pCsr->iCol==nCol ){
-      rc = FTS5_CORRUPT;
-    }
+    while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
+    assert( pCsr->iCol<pCsr->pConfig->nCol );
  }
  return rc;
 }
@@ -600,9 +505,7 @@ static int fts5VocabFilterMethod(
  int nUnused,                    /* Number of elements in apVal */
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
 ){
-  Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
-  int eType = pTab->eType;
  int rc = SQLITE_OK;

  int iVal = 0;
@@ -632,7 +535,6 @@ static int fts5VocabFilterMethod(
    }
    if( pLe ){
      const char *zCopy = (const char *)sqlite3_value_text(pLe);
-      if( zCopy==0 ) zCopy = "";
      pCsr->nLeTerm = sqlite3_value_bytes(pLe);
      pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1);
      if( pCsr->zLeTerm==0 ){
@@ -643,17 +545,11 @@ static int fts5VocabFilterMethod(
    }
  }

+
  if( rc==SQLITE_OK ){
-    Fts5Index *pIndex = pCsr->pFts5->pIndex;
-    rc = sqlite3Fts5IndexQuery(pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
+    rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
  }
-  if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){
-    rc = fts5VocabInstanceNewTerm(pCsr);
-  }
-  if( rc==SQLITE_OK && !pCsr->bEof 
-   && (eType!=FTS5_VOCAB_INSTANCE 
-    || pCsr->pFts5->pConfig->eDetail!=FTS5_DETAIL_NONE)
-  ){
+  if( rc==SQLITE_OK ){
    rc = fts5VocabNextMethod(pCursor);
  }

@@ -675,7 +571,7 @@ static int fts5VocabColumnMethod(
  int iCol                        /* Index of column to read value from */
 ){
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
-  int eDetail = pCsr->pFts5->pConfig->eDetail;
+  int eDetail = pCsr->pConfig->eDetail;
  int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType;
  i64 iVal = 0;

@@ -687,7 +583,7 @@ static int fts5VocabColumnMethod(
    assert( iCol==1 || iCol==2 || iCol==3 );
    if( iCol==1 ){
      if( eDetail!=FTS5_DETAIL_NONE ){
-        const char *z = pCsr->pFts5->pConfig->azCol[pCsr->iCol];
+        const char *z = pCsr->pConfig->azCol[pCsr->iCol];
        sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
      }
    }else if( iCol==2 ){
@@ -695,41 +591,13 @@ static int fts5VocabColumnMethod(
    }else{
      iVal = pCsr->aCnt[pCsr->iCol];
    }
-  }else if( eType==FTS5_VOCAB_ROW ){
+  }else{
    assert( iCol==1 || iCol==2 );
    if( iCol==1 ){
      iVal = pCsr->aDoc[0];
    }else{
      iVal = pCsr->aCnt[0];
    }
-  }else{
-    assert( eType==FTS5_VOCAB_INSTANCE );
-    switch( iCol ){
-      case 1:
-        sqlite3_result_int64(pCtx, pCsr->pIter->iRowid);
-        break;
-      case 2: {
-        int ii = -1;
-        if( eDetail==FTS5_DETAIL_FULL ){
-          ii = FTS5_POS2COLUMN(pCsr->iInstPos);
-        }else if( eDetail==FTS5_DETAIL_COLUMNS ){
-          ii = (int)pCsr->iInstPos;
-        }
-        if( ii>=0 && ii<pCsr->pFts5->pConfig->nCol ){
-          const char *z = pCsr->pFts5->pConfig->azCol[ii];
-          sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
-        }
-        break;
-      }
-      default: {
-        assert( iCol==3 );
-        if( eDetail==FTS5_DETAIL_FULL ){
-          int ii = FTS5_POS2OFFSET(pCsr->iInstPos);
-          sqlite3_result_int(pCtx, ii);
-        }
-        break;
-      }
-    }
  }

  if( iVal>0 ) sqlite3_result_int64(pCtx, iVal);
@@ -775,9 +643,10 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
    /* xSavepoint    */ 0,
    /* xRelease      */ 0,
    /* xRollbackTo   */ 0,
-    /* xShadowName   */ 0
  };
  void *p = (void*)pGlobal;

  return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
 }
+
+
@@ -89,6 +89,32 @@ input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); }
 %destructor expr     { sqlite3Fts5ParseNodeFree($$); }
 %destructor exprlist { sqlite3Fts5ParseNodeFree($$); }

+expr(A) ::= expr(X) AND expr(Y). {
+  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
+}
+expr(A) ::= expr(X) OR expr(Y). {
+  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
+}
+expr(A) ::= expr(X) NOT expr(Y). {
+  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
+}
+
+expr(A) ::= LP expr(X) RP. {A = X;}
+expr(A) ::= exprlist(X).   {A = X;}
+
+exprlist(A) ::= cnearset(X). {A = X;}
+exprlist(A) ::= exprlist(X) cnearset(Y). {
+  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
+}
+
+cnearset(A) ::= nearset(X). { 
+  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
+}
+cnearset(A) ::= colset(X) COLON nearset(Y). { 
+  sqlite3Fts5ParseSetColset(pParse, Y, X);
+  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
+}
+
 %type colset {Fts5Colset*}
 %destructor colset { sqlite3_free($$); }
 %type colsetlist {Fts5Colset*}
@@ -112,47 +138,12 @@ colsetlist(A) ::= STRING(X). {
  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
 }

-expr(A) ::= expr(X) AND expr(Y). {
-  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
-}
-expr(A) ::= expr(X) OR expr(Y). {
-  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
-}
-expr(A) ::= expr(X) NOT expr(Y). {
-  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
-}
-
-expr(A) ::= colset(X) COLON LP expr(Y) RP. {
-  sqlite3Fts5ParseSetColset(pParse, Y, X);
-  A = Y;
-}
-expr(A) ::= LP expr(X) RP. {A = X;}
-expr(A) ::= exprlist(X).   {A = X;}
-
-exprlist(A) ::= cnearset(X). {A = X;}
-exprlist(A) ::= exprlist(X) cnearset(Y). {
-  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
-}
-
-cnearset(A) ::= nearset(X). { 
-  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
-}
-cnearset(A) ::= colset(X) COLON nearset(Y). { 
-  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
-  sqlite3Fts5ParseSetColset(pParse, A, X);
-}
-
-
 %type nearset     {Fts5ExprNearset*}
 %type nearphrases {Fts5ExprNearset*}
 %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
 %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }

-nearset(A) ::= phrase(Y). { A = sqlite3Fts5ParseNearset(pParse, 0, Y); }
-nearset(A) ::= CARET phrase(Y). { 
-  sqlite3Fts5ParseSetCaret(Y);
-  A = sqlite3Fts5ParseNearset(pParse, 0, Y); 
-}
+nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
 nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
  sqlite3Fts5ParseNear(pParse, &X);
  sqlite3Fts5ParseSetDistance(pParse, Y, &Z);
@@ -193,5 +184,6 @@ phrase(A) ::= STRING(Y) star_opt(Z). {
 ** Optional "*" character.
 */
 %type star_opt {int}
+
 star_opt(A) ::= STAR. { A = 1; }
 star_opt(A) ::= . { A = 0; }
@@ -38,7 +38,8 @@ do_execsql_test 1.0 {
 do_execsql_test 1.1 {
  DROP TABLE t1;
  SELECT name, sql FROM sqlite_master;
-} {}
+} {
+}

 #-------------------------------------------------------------------------
 #
@@ -408,7 +409,6 @@ do_test 14.3 {
 do_execsql_test 15.0 {
  INSERT INTO t1(t1) VALUES('integrity-check');
 }
-sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 15.1 {
  UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1;
 }
@@ -427,7 +427,7 @@ proc funk {} {
  db eval { UPDATE n1_config SET v=50 WHERE k='version' }
  set fd [db incrblob main n1_data block 10]
  fconfigure $fd -encoding binary -translation binary
-#  puts -nonewline $fd "\x44\x45"
+  puts -nonewline $fd "\x44\x45"
  close $fd
 }
 db func funk funk
@@ -441,7 +441,7 @@ db func funk funk
 do_catchsql_test 16.2 {
  SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
 } {0 {{} -1e-06 {}}}
-# {1 {SQL logic error}}
+# {1 {SQL logic error or missing database}}

 #-------------------------------------------------------------------------
 #
@@ -561,81 +561,9 @@ do_test 20.1 {
  execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
 } $::ids

-#--------------------------------------------------------------------
-# Test that a DROP TABLE may be executed within a transaction that
-# writes to an FTS5 table.
-#
-do_execsql_test 21.0 {
-  CREATE TEMP TABLE t8(a, b);
-  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
-}
-
-do_execsql_test 21.1 {
-  BEGIN;
-    INSERT INTO ft VALUES('a b c');
-    DROP TABLE t8;
-  COMMIT;
-}
-
-do_execsql_test 22.0 {
-  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
-  INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
-  BEGIN;
-    INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
-    DELETE FROM t9 WHERE rowid = 2;
-    INSERT INTO t9(rowid, x) VALUES(3, 'bbb');
-  COMMIT;
-}
-
-do_execsql_test 22.1 {
-  SELECT rowid FROM t9('a*')
-} {1}
-
-#-------------------------------------------------------------------------
-do_execsql_test 23.0 {
-  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
-  CREATE TABLE t11(x);
-}
-do_execsql_test 23.1 {
-  SELECT * FROM t11, t10 WHERE t11.x = t10.x AND t10.rowid IS NULL;
-}
-do_execsql_test 23.2 {
-  SELECT * FROM t11, t10 WHERE t10.rowid IS NULL;
-}
-
-#-------------------------------------------------------------------------
-do_execsql_test 24.0 {
-  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%);
-  INSERT INTO t12 VALUES('aaaa');
-}
-do_execsql_test 24.1 {
-  BEGIN;
-    DELETE FROM t12 WHERE rowid=1;
-    SELECT * FROM t12('aaaa');
-    INSERT INTO t12 VALUES('aaaa');
-  END;
-}
-do_execsql_test 24.2 {
-  INSERT INTO t12(t12) VALUES('integrity-check');
-}
-do_execsql_test 24.3 {
-    SELECT * FROM t12('aaaa');
-} {aaaa}
-
-#-------------------------------------------------------------------------
-do_execsql_test 25.0 {
-  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%);
-}
-do_execsql_test 25.1 {
-  BEGIN;
-  INSERT INTO t13 VALUES('AAAA');
-SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB*');
-
-  END;
 }


-}
-
-expand_all_sql db
 finish_test
+
+
@@ -294,3 +294,4 @@ do_execsql_test 7.0 {


 finish_test
+
@@ -276,3 +276,4 @@ foreach {tn expr tclexpr} {
 }

 finish_test
+
@@ -231,6 +231,7 @@ foreach {T create} {
        set res [lsort -integer -increasing $res]
      }
      set n [llength $res]
+      if {$T==5} breakpoint 
      do_execsql_test $T.$bAsc.$tn.$n $sql $res
    }
  }
@@ -241,3 +242,4 @@ foreach {T create} {
 }

 finish_test
+
@@ -309,3 +309,4 @@ foreach {tn q cnt} {
 }

 finish_test
+
@@ -175,16 +175,7 @@ do_execsql_test 5.1 {
  SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
 } {{[x] a a a a a...}}

-do_execsql_test 5.2 {
-  SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x');
-} {{[x] a a a a a}}
-do_execsql_test 5.3 {
-  SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x');
-} {{x] a a a a a...}}
-do_execsql_test 5.4 {
-  SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x');
-} {{[x a a a a a...}}
-
 } ;# foreach_detail_mode 

 finish_test
+
@@ -142,3 +142,4 @@ if {[detail_is_full]} {


 finish_test
+
@@ -167,3 +167,4 @@ do_execsql_test 1.8.2 {
 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}

 finish_test
+
@@ -51,13 +51,8 @@ do_execsql_test 1.1 {
 do_execsql_test 1.2 {
  INSERT INTO t1(t1) VALUES('integrity-check');
 }
-
-do_execsql_test 1.3 {
-  SAVEPOINT one;
-    INSERT INTO t1 VALUES('v w x');
-  ROLLBACK TO one;
-}
 }


 finish_test
+
@@ -66,3 +66,4 @@ do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }


 finish_test
+
@@ -144,14 +144,7 @@ do_execsql_test 3.1 {
  {[a b c d e]}
 }

-do_execsql_test 3.2 {
-  SELECT highlight(ft, 0, NULL, NULL) FROM ft WHERE ft MATCH 'a+b+c AND c+d+e';
-} {
-  {a b c x c d e}
-  {a b c c d e}
-  {a b c d e}
-}
-
 }

 finish_test
+
@@ -77,7 +77,7 @@ foreach {tn defn} {
 } {
  do_test 2.2.$tn {
    catchsql { INSERT INTO ft1(ft1, rank) VALUES('rank', $defn) }
-  } {1 {SQL logic error}}
+  } {1 {SQL logic error or missing database}}
 }

 #-------------------------------------------------------------------------
@@ -297,3 +297,4 @@ do_catchsql_test 4.4.4 {


 finish_test
+
@@ -89,6 +89,7 @@ do_execsql_test 3.1 {
  BEGIN;
    INSERT INTO abc(rowid, a) VALUES(2, 'a');
 }
+breakpoint
 do_execsql_test 3.2 {
    SELECT rowid FROM abc WHERE abc MATCH 'a';
 } {1 2}
@@ -99,3 +100,4 @@ do_execsql_test 3.3 {
 } {1 2}

 finish_test
+
@@ -342,3 +342,4 @@ foreach {tn expr} {


 finish_test
+
@@ -240,6 +240,7 @@ foreach {tn lRow res} {
 } {
  execsql { DELETE FROM x1 }
  foreach row $lRow { execsql { INSERT INTO x1 VALUES($row) } }
+  breakpoint
  do_execsql_test 8.$tn {
    SELECT highlight(x1, 0, '[', ']') FROM x1 WHERE x1 MATCH 'a OR (b AND d)';
  } $res
@@ -275,37 +276,7 @@ do_execsql_test 9.3 {
  9 10
 }

-#-------------------------------------------------------------------------
-# Test that aux. functions may not be used in aggregate queries.
-#
-reset_db
-do_execsql_test 10.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, z);
-  INSERT INTO t1 VALUES('a', 'one two', 1);
-  INSERT INTO t1 VALUES('b', 'two three', 2);
-  INSERT INTO t1 VALUES('c', 'three four', 1);
-  INSERT INTO t1 VALUES('d', 'four five', 2);
-  INSERT INTO t1 VALUES('e', 'five six', 1);
-  INSERT INTO t1 VALUES('f', 'six seven', 2);
-}

-proc firstcol {cmd} { $cmd xColumnText 0 }
-sqlite3_fts5_create_function db firstcol firstcol
-
-do_execsql_test 10.1.1 {
-  SELECT firstcol(t1) FROM t1
-} {a b c d e f}
-do_execsql_test 10.1.2 {
-  SELECT group_concat(x, '.') FROM t1
-} {a.b.c.d.e.f}
-
-do_catchsql_test 10.1.3 {
-  SELECT group_concat(firstcol(t1), '.') FROM t1
-} {1 {unable to use function firstcol in the requested context}}
-
-do_catchsql_test 10.1.4 {
-  SELECT group_concat(firstcol(t1), '.') FROM t1 GROUP BY rowid
-} {1 {unable to use function firstcol in the requested context}}

 finish_test

@@ -112,3 +112,4 @@ db eval {
 }

 finish_test
+
@@ -61,3 +61,4 @@ do_test 2.1...slow {
 } {}

 finish_test
+
@@ -64,3 +64,5 @@ foreach_detail_mode $::testprefix {
 }

 finish_test
+
+
@@ -1,59 +0,0 @@
-# 2016 Jan 15
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-ifcapable !fts5 { finish_test ; return }
-set ::testprefix fts5cat
-
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize="unicode61 categories 'L*'");
-  INSERT INTO t1 VALUES ('Unlike1option2values3and4column5names');
-}
-
-do_execsql_test 1.1 {
-  SELECT rowid FROM t1('option');
-} {1}
-
-do_execsql_test 1.2 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x);
-  CREATE VIRTUAL TABLE t2t USING fts5vocab(t2, row);
-
-  CREATE VIRTUAL TABLE t3 USING fts5(
-    x, tokenize="unicode61 categories 'L* N* Co Mn'"
-  );
-  CREATE VIRTUAL TABLE t3t USING fts5vocab(t3, row);
-
-  CREATE VIRTUAL TABLE t4 USING fts5(
-    x, tokenize="unicode61 categories 'L* N* Co M*'"
-  );
-  CREATE VIRTUAL TABLE t4t USING fts5vocab(t4, row);
-
-  INSERT INTO t2 VALUES ('สนามกีฬา');
-  INSERT INTO t3 VALUES ('สนามกีฬา');
-  INSERT INTO t4 VALUES ('สนามกีฬา');
-}
-
-do_execsql_test 1.3 {
-  SELECT * FROM t2t
-} {สนามก 1 1 ฬา 1 1}
-
-do_execsql_test 1.4 {
-  SELECT * FROM t3t
-} {สนามกีฬา 1 1}
-
-do_execsql_test 1.5 {
-  SELECT * FROM t4t
-} {สนามกีฬา 1 1}
-
-
-finish_test
@@ -1,80 +0,0 @@
-# 2018 Dec 22
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS5 module.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5circref
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE tt USING fts5(a);
-  SELECT name FROM sqlite_master ORDER BY 1;
-} {
-  tt tt_config tt_content tt_data tt_docsize tt_idx
-}
-db_save_and_close
-
-foreach {tn schema sql} {
-  1 {
-    CREATE TRIGGER tr1 AFTER INSERT ON tt_config BEGIN
-      SELECT * FROM tt;
-    END;
-  } {
-    INSERT INTO tt(tt, rank) VALUES('usermerge', 4);
-  }
-
-  2 {
-    CREATE TRIGGER tr1 AFTER INSERT ON tt_docsize BEGIN
-      SELECT * FROM tt;
-    END;
-  } {
-    INSERT INTO tt(a) VALUES('one two three');
-  }
-
-  3 {
-    CREATE TRIGGER tr1 AFTER INSERT ON tt_content BEGIN
-      SELECT * FROM tt;
-    END;
-  } {
-    INSERT INTO tt(a) VALUES('one two three');
-  }
-
-  4 {
-    CREATE TRIGGER tr1 AFTER INSERT ON tt_data BEGIN
-      SELECT * FROM tt;
-    END;
-  } {
-    INSERT INTO tt(a) VALUES('one two three');
-  }
-
-  5 {
-    CREATE TRIGGER tr1 AFTER INSERT ON tt_idx BEGIN
-      SELECT * FROM tt;
-    END;
-  } {
-    INSERT INTO tt(a) VALUES('one two three');
-  }
-} {
-  db_restore_and_reopen
-  do_execsql_test 1.1.$tn.1 $schema
-  do_catchsql_test 1.1.$tn.2 $sql {1 {SQL logic error}}
-  db close
-}
-
-
-finish_test
@@ -44,43 +44,16 @@ foreach_detail_mode $::testprefix {
    9 "-c : a"  {1 2 4}
    10 "-\"c\" : a"  {1 2 4}
  } {
+  breakpoint
    do_execsql_test 1.$tn {
      SELECT rowid FROM t1($q)
    } $res
  }

-  foreach {tn q res} {
-    0 {{a} : (a AND ":")}     {}
-    1 "{a b c} : (a AND d)"   {2 3}
-    2 "{a b c} : (a AND b:d)" {3}
-    3 "{a b c} : (a AND d:d)" {}
-    4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
-    5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
-    6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
-    7 "{a b c} : (b:a AND c:b)" {2}
-  } {
-    do_execsql_test 2.$tn {
-      SELECT rowid FROM t1($q)
-    } $res
-  }

-  foreach {tn w res} {
-    0 "a MATCH 'a'" {1}
-    1 "b MATCH 'a'" {2}
-    2 "b MATCH '{a b c} : a'" {2}
-    3 "b MATCH 'a OR b'"      {1 2}
-    4 "b MATCH 'a OR a:b'"    {2}
-    5 "b MATCH 'a OR b:b'"    {1 2}
-  } {
-    do_execsql_test 3.$tn "
-      SELECT rowid FROM t1 WHERE $w
-    " $res
-  }
-
-  do_catchsql_test 4.1 {
-    SELECT * FROM t1 WHERE rowid MATCH 'a'
-  } {1 {unable to use function MATCH in the requested context}}
 }


 finish_test
+
+
@@ -143,6 +143,7 @@ do_execsql_test 4.1.1 {
  INSERT INTO t5 VALUES('2 4 6 8');
 }

+breakpoint
 do_execsql_test 4.1.2 {
  INSERT INTO t5(t5) VALUES('integrity-check');
 }
@@ -66,7 +66,7 @@ foreach {tn val} {
 } {
  do_catchsql_test 3.$tn {
    INSERT INTO t1(t1, rank) VALUES('rank', $val);
-  } {1 {SQL logic error}}
+  } {1 {SQL logic error or missing database}}
 }

 #-------------------------------------------------------------------------
@@ -110,6 +110,7 @@ do_catchsql_test 5.1 {
  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize="porter 'ascii");
 } {1 {parse error in tokenize directive}} 

+breakpoint
 do_catchsql_test 5.2 {
  CREATE VIRTUAL TABLE xx USING fts5(x, [y[]);
 } {0 {}}
@@ -168,33 +169,33 @@ do_execsql_test 9.0 {
 } {}
 do_catchsql_test 9.1.1 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', -5);
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}
 do_catchsql_test 9.1.2 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000);
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}
 do_catchsql_test 9.1.3 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67);
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}

 do_catchsql_test 9.2.1 {
  INSERT INTO abc(abc, rank) VALUES('automerge', -5);
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}
 do_catchsql_test 9.2.2 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 50000000);
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}
 do_catchsql_test 9.2.3 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 66.67);
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}
 do_execsql_test 9.2.4 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 1);
 } {}

 do_catchsql_test 9.3.1 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', -5);
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}
 do_catchsql_test 9.3.2 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67);
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}
 do_execsql_test 9.3.3 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 1);
 } {}
@@ -204,14 +205,14 @@ do_execsql_test 9.3.4 {

 do_catchsql_test 9.4.1 {
  INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}

 do_catchsql_test 9.5.1 {
  INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}
 do_catchsql_test 9.5.2 {
  INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}
 do_catchsql_test 9.5.3 {
  INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
 } {0 {}}
@@ -244,7 +245,7 @@ foreach {tn opt} {

 do_catchsql_test 12.1 {
  INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
-} {1 {SQL logic error}}
+} {1 {SQL logic error or missing database}}

 #-------------------------------------------------------------------------
 # errors in the 'usermerge' option
@@ -259,7 +260,8 @@ foreach {tn val} {
  4     1
 } {
  set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)"
-  do_catchsql_test 13.$tn $sql {1 {SQL logic error}}
+  do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}}
 }

 finish_test
+
@@ -66,3 +66,5 @@ do_execsql_test 2.1 {
 }

 finish_test
+
+
@@ -1,246 +0,0 @@
-# 2017 August 17
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-#
-
-
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5connect
-
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-#-------------------------------------------------------------------------
-# The tests in this file test the outcome of a schema-reset happening 
-# within the xConnect() method of an FTS5 table. At one point this
-# was causing a problem in SQLite. Each test proceeds as follows:
-#
-#   1. Connection [db] opens the db and reads from some unrelated, non-FTS5
-#      table causing SQLite to load the db schema into memory.
-#
-#   2. Connection [db2] opens the db and modifies the db schema.
-#
-#   3. Connection [db] reads or writes an existing fts5 table. That the
-#      schema has been modified is detected inside the fts5 xConnect() 
-#      callback that is invoked by sqlite3_prepare(). 
-#
-#   4. Verify that the statement in 3 has worked. SQLite should detect
-#      that the schema has changed and successfully prepare the 
-#      statement against the new schema.
-#
-# Test plan:
-#
-#   1.*: Trigger the xConnect()/schema-reset using statements executed
-#        directly against an FTS5 table.
-#
-#   2.*: Using various statements executed by various BEFORE triggers.
-#
-#   3.*: Using various statements executed by various AFTER triggers.
-#
-#   4.*: Using various statements executed by various INSTEAD OF triggers.
-#
-
-
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE ft1 USING fts5(a, b);
-  CREATE TABLE abc(x INTEGER PRIMARY KEY);
-  CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b);
-
-  INSERT INTO ft1 VALUES('one', 'two');
-  INSERT INTO ft1 VALUES('three', 'four');
-}
-
-foreach {tn sql res} {
-  1 "SELECT * FROM ft1" {one two three four}
-  2 "REPLACE INTO ft1(rowid, a, b) VALUES(1, 'five', 'six')" {}
-  3 "SELECT * FROM ft1" {five six three four}
-  4 "INSERT INTO ft1 VALUES('seven', 'eight')" {}
-  5 "SELECT * FROM ft1" {five six three four seven eight}
-  6 "DELETE FROM ft1 WHERE rowid=2" {}
-  7 "UPDATE ft1 SET b='nine' WHERE rowid=1" {}
-  8 "SELECT * FROM ft1" {five nine seven eight}
-} {
-
-  catch { db close }
-  catch { db2 close }
-  sqlite3 db  test.db
-  sqlite3 db2 test.db
-
-  do_test 1.$tn.1 {
-    db eval { INSERT INTO abc DEFAULT VALUES }
-    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
-  } {}
-
-  do_execsql_test 1.$tn.2 $sql $res
-
-  do_execsql_test 1.$tn.3 {
-    INSERT INTO ft1(ft1) VALUES('integrity-check');
-  }
-}
-
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE ft2 USING fts5(a, b);
-  CREATE TABLE t2(a, b);
-  CREATE TABLE log(txt);
-
-  CREATE TRIGGER t2_ai AFTER INSERT ON t2 BEGIN
-    INSERT INTO ft2(rowid, a, b) VALUES(new.rowid, new.a, new.b);
-    INSERT INTO log VALUES('insert');
-  END;
-
-  CREATE TRIGGER t2_ad AFTER DELETE ON t2 BEGIN
-    DELETE FROM ft2 WHERE rowid = old.rowid;
-    INSERT INTO log VALUES('delete');
-  END;
-
-  CREATE TRIGGER t2_au AFTER UPDATE ON t2 BEGIN
-    UPDATE ft2 SET a=new.a, b=new.b WHERE rowid=new.rowid;
-    INSERT INTO log VALUES('update');
-  END;
-
-  INSERT INTO t2 VALUES('one', 'two');
-  INSERT INTO t2 VALUES('three', 'four');
-}
-
-foreach {tn sql res} {
-  1 "SELECT * FROM t2" {one two three four}
-  2 "REPLACE INTO t2(rowid, a, b) VALUES(1, 'five', 'six')" {}
-  3 "SELECT * FROM ft2" {five six three four}
-  4 "INSERT INTO t2 VALUES('seven', 'eight')" {}
-  5 "SELECT * FROM ft2" {five six three four seven eight}
-  6 "DELETE FROM t2 WHERE rowid=2" {}
-  7 "UPDATE t2 SET b='nine' WHERE rowid=1" {}
-  8 "SELECT * FROM ft2" {five nine seven eight}
-} {
-
-  catch { db close }
-  catch { db2 close }
-  sqlite3 db  test.db
-  sqlite3 db2 test.db
-
-  do_test 2.$tn.1 {
-    db eval { INSERT INTO abc DEFAULT VALUES }
-    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
-  } {}
-
-  do_execsql_test 2.$tn.2 $sql $res
-
-  do_execsql_test 2.$tn.3 {
-    INSERT INTO ft2(ft2) VALUES('integrity-check');
-  }
-}
-
-do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE ft3 USING fts5(a, b);
-  CREATE TABLE t3(a, b);
-
-  CREATE TRIGGER t3_ai BEFORE INSERT ON t3 BEGIN
-    INSERT INTO ft3(rowid, a, b) VALUES(new.rowid, new.a, new.b);
-    INSERT INTO log VALUES('insert');
-  END;
-
-  CREATE TRIGGER t3_ad BEFORE DELETE ON t3 BEGIN
-    DELETE FROM ft3 WHERE rowid = old.rowid;
-    INSERT INTO log VALUES('delete');
-  END;
-
-  CREATE TRIGGER t3_au BEFORE UPDATE ON t3 BEGIN
-    UPDATE ft3 SET a=new.a, b=new.b WHERE rowid=new.rowid;
-    INSERT INTO log VALUES('update');
-  END;
-
-  INSERT INTO t3(rowid, a, b) VALUES(1, 'one', 'two');
-  INSERT INTO t3(rowid, a, b) VALUES(2, 'three', 'four');
-}
-
-foreach {tn sql res} {
-  1 "SELECT * FROM t3" {one two three four}
-  2 "REPLACE INTO t3(rowid, a, b) VALUES(1, 'five', 'six')" {}
-  3 "SELECT * FROM ft3" {five six three four}
-  4 "INSERT INTO t3(rowid, a, b) VALUES(3, 'seven', 'eight')" {}
-  5 "SELECT * FROM ft3" {five six three four seven eight}
-  6 "DELETE FROM t3 WHERE rowid=2" {}
-  7 "UPDATE t3 SET b='nine' WHERE rowid=1" {}
-  8 "SELECT * FROM ft3" {five nine seven eight}
-} {
-
-  catch { db close }
-  catch { db2 close }
-  sqlite3 db  test.db
-  sqlite3 db2 test.db
-
-  do_test 3.$tn.1 {
-    db eval { INSERT INTO abc DEFAULT VALUES }
-    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
-  } {}
-
-  do_execsql_test 3.$tn.2 $sql $res
-
-  do_execsql_test 3.$tn.3 {
-    INSERT INTO ft3(ft3) VALUES('integrity-check');
-  }
-}
-
-do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE ft4 USING fts5(a, b);
-  CREATE VIEW v4 AS SELECT rowid, * FROM ft4;
-
-  CREATE TRIGGER t4_ai INSTEAD OF INSERT ON v4 BEGIN
-    INSERT INTO ft4(rowid, a, b) VALUES(new.rowid, new.a, new.b);
-    INSERT INTO log VALUES('insert');
-  END;
-
-  CREATE TRIGGER t4_ad INSTEAD OF DELETE ON v4 BEGIN
-    DELETE FROM ft4 WHERE rowid = old.rowid;
-    INSERT INTO log VALUES('delete');
-  END;
-
-  CREATE TRIGGER t4_au INSTEAD OF UPDATE ON v4 BEGIN
-    UPDATE ft4 SET a=new.a, b=new.b WHERE rowid=new.rowid;
-    INSERT INTO log VALUES('update');
-  END;
-
-  INSERT INTO ft4(rowid, a, b) VALUES(1, 'one', 'two');
-  INSERT INTO ft4(rowid, a, b) VALUES(2, 'three', 'four');
-}
-
-foreach {tn sql res} {
-  1 "SELECT * FROM ft4" {one two three four}
-  2 "REPLACE INTO v4(rowid, a, b) VALUES(1, 'five', 'six')" {}
-  3 "SELECT * FROM ft4" {five six three four}
-  4 "INSERT INTO v4(rowid, a, b) VALUES(3, 'seven', 'eight')" {}
-  5 "SELECT * FROM ft4" {five six three four seven eight}
-  6 "DELETE FROM v4 WHERE rowid=2" {}
-  7 "UPDATE v4 SET b='nine' WHERE rowid=1" {}
-  8 "SELECT * FROM ft4" {five nine seven eight}
-} {
-
-  catch { db close }
-  catch { db2 close }
-  sqlite3 db  test.db
-  sqlite3 db2 test.db
-
-  do_test 4.$tn.1 {
-    db eval { INSERT INTO abc DEFAULT VALUES }
-    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
-  } {}
-
-  do_execsql_test 4.$tn.2 $sql $res
-
-  do_execsql_test 4.$tn.3 {
-    INSERT INTO ft3(ft3) VALUES('integrity-check');
-  }
-}
-
-finish_test
@@ -253,45 +253,6 @@ do_execsql_test 6.2 {
  SELECT name FROM sqlite_master;
 } {}

-#---------------------------------------------------------------------------
-# Check that an fts5 table cannot be its own content table.
-#
-reset_db
-do_execsql_test 7.1.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, c=t1 );
-  INSERT INTO t1( a ) VALUES('abc');
-}
-do_catchsql_test 7.1.2 { 
-  SELECT * FROM t1; 
-} {1 {recursively defined fts5 content table}}
-do_catchsql_test 7.1.3 { 
-  SELECT * FROM t1('abc'); 
-} {1 {recursively defined fts5 content table}}
-do_catchsql_test 7.1.4 { 
-  SELECT count(*) FROM t1;
-} {1 {recursively defined fts5 content table}}
-do_catchsql_test 7.1.5 { 
-  SELECT * FROM t1('abc') ORDER BY rank;
-} {1 {recursively defined fts5 content table}}
-
-reset_db
-do_execsql_test 7.2.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, c=t2 );
-  CREATE VIRTUAL TABLE t2 USING fts5(a, c=t1 );
-  INSERT INTO t1( a ) VALUES('abc');
-}
-do_catchsql_test 7.2.2 { 
-  SELECT * FROM t1; 
-} {1 {recursively defined fts5 content table}}
-do_catchsql_test 7.2.3 { 
-  SELECT * FROM t1('abc'); 
-} {1 {recursively defined fts5 content table}}
-do_catchsql_test 7.2.4 { 
-  SELECT count(*) FROM t1;
-} {1 {recursively defined fts5 content table}}
-do_catchsql_test 7.2.5 { 
-  SELECT * FROM t1('abc') ORDER BY rank;
-} {1 {recursively defined fts5 content table}}

 finish_test

@@ -41,7 +41,6 @@ db_save
 do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 set segid [lindex [fts5_level_segids t1] 0]

-sqlite3_db_config db DEFENSIVE 0
 do_test 1.3 {
  execsql {
    DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4);
@@ -51,7 +50,6 @@ do_test 1.3 {

 do_test 1.4 {
  db_restore_and_reopen
-  sqlite3_db_config db DEFENSIVE 0
  execsql {
    UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE
    rowid = fts5_rowid('segment', $segid, 4);
@@ -91,10 +89,11 @@ do_execsql_test 3.0 {
 do_execsql_test 3.1 {
  SELECT * FROM t3 WHERE t3 MATCH 'o'
 } {{one o} {three o} {five o}}
-sqlite3_db_config db DEFENSIVE 0
+
 do_catchsql_test 3.1 {
  DELETE FROM t3_content WHERE rowid = 3;
  SELECT * FROM t3 WHERE t3 MATCH 'o';
 } {1 {database disk image is malformed}}

 finish_test
+
@@ -99,7 +99,6 @@ foreach {tno stmt} {
 set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] 
 set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
 set all [db eval {SELECT rowid FROM t1}]
-sqlite3_db_config db DEFENSIVE 0
 for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
  do_execsql_test 2.$i.1 {
    BEGIN;
@@ -249,7 +248,6 @@ foreach {tn hdr} {

 #--------------------------------------------------------------------
 reset_db
-sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE x5 USING fts5(tt);
  INSERT INTO x5 VALUES('a');
@@ -271,3 +269,4 @@ do_catchsql_test 6.2 {

 sqlite3_fts5_may_be_corrupt 0
 finish_test
+
@@ -1,61 +0,0 @@
-# 2019 May 16
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5corrupt4
-
-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-sqlite3_fts5_may_be_corrupt 1
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE ttt USING fts5(a, b); 
-  INSERT INTO ttt 
-     VALUES('e ee eee e ee eee e ee eee', 'eee ee e e e ee eee ee ee');
-  INSERT INTO ttt SELECT a||a, b||b FROM ttt;   
-  INSERT INTO ttt SELECT a||a, b||b FROM ttt;   
-}
-
-proc mutate {blob i} {
-  set o [expr {$i % [string length $blob]}]
-  set a [string range $blob 0 $o-1]
-  set b [string range $blob $o+1 end]
-  set v [expr int(rand()*255) - 127]
-  return "$a[binary format c $v]$b"
-}
-db func mutate mutate
-
-for {set j 1000} {$j <= 5000} {incr j 1000} {
-  do_test 1.$j {
-    for {set i 0} {$i < 1000} {incr i} {
-      execsql {
-        BEGIN;
-          UPDATE ttt_data SET block = mutate(block, $i) WHERE id>10;
-      }
-      foreach sql {
-        {SELECT snippet(ttt, -1, '.', '..', '[', ']'), * FROM ttt('e*')}
-        {SELECT snippet(ttt, -1, '.', '..', '[', ']'), * FROM ttt('e* NOT ee*')}
-      } {
-        catch { execsql $sql }
-      }
-      execsql ROLLBACK
-    }
-  } {}
-}
-
-sqlite3_fts5_may_be_corrupt 0
-finish_test
-
@@ -1,53 +0,0 @@
-# 2017 May 12
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS5 module.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5delete
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-fts5_aux_test_functions db
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x);
-  WITH s(i) AS (
-    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<5000
-  )
-  INSERT INTO t1(rowid, x) SELECT i, (i/2)*2 FROM s;
-}
-
-do_test 1.1 {
-  execsql BEGIN
-  for {set i 1} {$i<=5000} {incr i} {
-    if {$i % 2} {
-      execsql { INSERT INTO t1 VALUES($i) }
-    } else {
-      execsql { DELETE FROM t1 WHERE rowid = $i }
-    }
-  }
-  execsql COMMIT
-} {}
-
-do_test 1.2 {
-  execsql { INSERT INTO t1(t1, rank) VALUES('usermerge', 2); }
-  for {set i 0} {$i < 5} {incr i} {
-    execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
-    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
-  }
-} {}
-
-finish_test
@@ -241,3 +241,4 @@ do_execsql_test 5.3 {


 finish_test
+
@@ -63,3 +63,5 @@ foreach_detail_mode $::testprefix {


 finish_test
+
+
@@ -66,6 +66,7 @@ proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} {
  }
  execsql COMMIT

+  breakpoint
  do_test $tn.1 {
    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
  } {}
@@ -123,6 +124,7 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} {
  do_execsql_test $tn.1 {
    SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a'
  } {1}
+  breakpoint
  do_execsql_test $tn.2 {
    SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
  } {1}
@@ -195,3 +197,4 @@ foreach v $vocab {


 finish_test
+
@@ -44,3 +44,4 @@ do_execsql_test 1.2 {


 finish_test
+
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .32.0
 .16.0
				`@@ -66,3 +66,4 @@ do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }`


				`finish_test`
				`@@ -63,3 +63,5 @@ foreach_detail_mode $::testprefix {`


				`finish_test`