Add code to get the tokenizer character-class logic working for EBCDIC.

FossilOrigin-Name: 04f7da77c13925c1f1e287f4579bb85518297d81
Faster keywordCode() implementation by taking advantage of the fact that
2016-02-08 19:15:48 +00:00 · 2016-02-08 03:23:46 +00:00 · 2016-02-08 02:30:50 +00:00 · 2016-02-07 00:08:08 +00:00 · 2016-02-06 22:32:06 +00:00 · 2016-02-06 19:48:50 +00:00
346 changed files with 22367 additions and 23728 deletions
@@ -1,138 +0,0 @@
-#!/usr/make
-#
-# Makefile for SQLITE
-#
-# This is a template makefile for SQLite.  Most people prefer to
-# use the autoconf generated "configure" script to generate the
-# makefile automatically.  But that does not work for everybody
-# and in every situation.  If you are having problems with the
-# "configure" script, you might want to try this makefile as an
-# alternative.  Create a copy of this file, edit the parameters
-# below and type "make".
-#
-
-#### The directory where to find the mingw32ce tools
-MINGW32CE = /opt/mingw32ce/bin
-
-#### The target prefix of the mingw32ce tools
-TARGET = arm-wince-mingw32ce
-
-#### The toplevel directory of the source tree.  This is the directory
-#    that contains this "Makefile.in" and the "configure.in" script.
-#
-TOP = ../sqlite
-
-#### C Compiler and options for use in building executables that
-#    will run on the platform that is doing the build.
-#
-BCC = gcc -g -O2
-#BCC = /opt/ancic/bin/c89 -0
-
-#### If the target operating system supports the "usleep()" system
-#    call, then define the HAVE_USLEEP macro for all C modules.
-#
-USLEEP = 
-#USLEEP = -DHAVE_USLEEP=1
-
-#### If you want the SQLite library to be safe for use within a 
-#    multi-threaded program, then define the following macro
-#    appropriately:
-#
-THREADSAFE = -DTHREADSAFE=1
-#THREADSAFE = -DTHREADSAFE=0
-
-#### Specify any extra linker options needed to make the library
-#    thread safe
-#
-#THREADLIB = -lpthread
-THREADLIB = 
-
-#### Specify any extra libraries needed to access required functions.
-#
-#TLIBS = -lrt    # fdatasync on Solaris 8
-TLIBS = 
-
-#### Leave SQLITE_DEBUG undefined for maximum speed.  Use SQLITE_DEBUG=1
-#    to check for memory leaks.  Use SQLITE_DEBUG=2 to print a log of all
-#    malloc()s and free()s in order to track down memory leaks.
-#    
-#    SQLite uses some expensive assert() statements in the inner loop.
-#    You can make the library go almost twice as fast if you compile
-#    with -DNDEBUG=1
-#
-#OPTS = -DSQLITE_DEBUG=2
-#OPTS = -DSQLITE_DEBUG=1
-#OPTS = 
-OPTS = -DNDEBUG=1 -DSQLITE_OS_WIN=1 -D_WIN32_WCE=1
-#OPTS += -DHAVE_FDATASYNC=1
-
-#### The suffix to add to executable files.  ".exe" for windows.
-#    Nothing for unix.
-#
-EXE = .exe
-#EXE =
-
-#### C Compile and options for use in building executables that 
-#    will run on the target platform.  This is usually the same
-#    as BCC, unless you are cross-compiling.
-#
-#TCC = gcc -O6
-#TCC = gcc -g -O0 -Wall
-#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage
-#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6
-TCC = $(MINGW32CE)/$(TARGET)-gcc -O2
-#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive
-
-#### Tools used to build a static library.
-#
-#AR = ar cr
-#AR = /opt/mingw/bin/i386-mingw32-ar cr
-AR = $(MINGW32CE)/$(TARGET)-ar cr
-#RANLIB = ranlib
-#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib
-RANLIB = $(MINGW32CE)/$(TARGET)-ranlib
-
-#MKSHLIB = gcc -shared
-#SO = so
-#SHPREFIX = lib
-MKSHLIB = $(MINGW32CE)/$(TARGET)-gcc -shared
-SO = dll
-SHPREFIX =
-
-#### Extra compiler options needed for programs that use the TCL library.
-#
-#TCL_FLAGS =
-#TCL_FLAGS = -DSTATIC_BUILD=1
-TCL_FLAGS = -I/home/drh/tcltk/8.5linux
-#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1
-#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux
-
-#### Linker options needed to link against the TCL library.
-#
-#LIBTCL = -ltcl -lm -ldl
-LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl
-#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt
-#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc
-
-#### Additional objects for SQLite library when TCL support is enabled.
-TCLOBJ =
-#TCLOBJ = tclsqlite.o
-
-#### Compiler options needed for programs that use the readline() library.
-#
-READLINE_FLAGS =
-#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline
-
-#### Linker options needed by programs using readline() must link against.
-#
-LIBREADLINE =
-#LIBREADLINE = -static -lreadline -ltermcap
-
-#### Which "awk" program provides nawk compatibilty
-#
-# NAWK = nawk
-NAWK = awk
-
-# You should not have to change anything below this line
-###############################################################################
-include $(TOP)/main.mk
@@ -54,6 +54,7 @@ LIBTCL = @TCL_LIB_SPEC@
 # Compiler options needed for programs that use the readline() library.
 #
 READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@
+READLINE_FLAGS += -DHAVE_EDITLINE=@TARGET_HAVE_EDITLINE@

 # The library that programs using readline() must link against.
 #
@@ -155,9 +156,6 @@ LTCOMPILE = $(LIBTOOL) --mode=compile --tag=CC $(TCC) $(LTCOMPILE_EXTRAS)
 LTLINK = $(LIBTOOL) --mode=link $(TCC) $(LTCOMPILE_EXTRAS) @LDFLAGS@ $(LTLINK_EXTRAS)
 LTINSTALL = $(LIBTOOL) --mode=install $(INSTALL)

-# nawk compatible awk.
-NAWK = @AWK@
-
 # You should not have to change anything below this line
 ###############################################################################

@@ -181,7 +179,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
         mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
         notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
         pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
-         random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
+         random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
         table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
         update.lo util.lo vacuum.lo \
         vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
@@ -404,6 +402,7 @@ TESTSRC = \
  $(TOP)/src/test_tclvar.c \
  $(TOP)/src/test_thread.c \
  $(TOP)/src/test_vfs.c \
+  $(TOP)/src/test_windirent.c \
  $(TOP)/src/test_wsd.c       \
  $(TOP)/ext/fts3/fts3_term.c \
  $(TOP)/ext/fts3/fts3_test.c \
@@ -419,6 +418,7 @@ TESTSRC += \
  $(TOP)/ext/misc/fuzzer.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
  $(TOP)/ext/fts5/fts5_test_mi.c \
+  $(TOP)/ext/fts5/fts5_test_tok.c \
  $(TOP)/ext/misc/ieee754.c \
  $(TOP)/ext/misc/nextchar.c \
  $(TOP)/ext/misc/percentile.c \
@@ -551,7 +551,7 @@ TESTOPTS = --verbose=file --output=test-out.txt
 #
 SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1
+FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5

 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
@@ -583,6 +583,12 @@ sqlite3$(TEXE):	$(TOP)/src/shell.c libsqlite3.la sqlite3.h
 sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)

+srcck1$(BEXE):	$(TOP)/tool/srcck1.c
+	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
+
+sourcetest:	srcck1$(BEXE) sqlite3.c
+	./srcck1 sqlite3.c
+
 fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
 	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
@@ -647,9 +653,9 @@ sqlite3.lo:	sqlite3.c

 # Rules to build the LEMON compiler generator
 #
-lemon$(BEXE):	$(TOP)/tool/lemon.c $(TOP)/src/lempar.c
+lemon$(BEXE):	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
 	$(BCC) -o $@ $(TOP)/tool/lemon.c
-	cp $(TOP)/src/lempar.c .
+	cp $(TOP)/tool/lempar.c .

 # Rules to build individual *.o files from generated *.c files. This
 # applies to:
@@ -902,22 +908,22 @@ tclsqlite3$(TEXE):	tclsqlite-shell.lo libsqlite3.la

 # Rules to build opcodes.c and opcodes.h
 #
-opcodes.c:	opcodes.h $(TOP)/mkopcodec.awk
-	$(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c
+opcodes.c:	opcodes.h $(TOP)/tool/mkopcodec.tcl
+	$(TCLSH_CMD) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c

-opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
-	cat parse.h $(TOP)/src/vdbe.c | $(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h
+opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl
+	cat parse.h $(TOP)/src/vdbe.c | $(TCLSH_CMD) $(TOP)/tool/mkopcodeh.tcl >opcodes.h

 # Rules to build parse.c and parse.h - the outputs of lemon.
 #
 parse.h:	parse.c

-parse.c:	$(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk
+parse.c:	$(TOP)/src/parse.y lemon$(BEXE) $(TOP)/tool/addopcodes.tcl
 	cp $(TOP)/src/parse.y .
 	rm -f parse.h
 	./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
 	mv parse.h parse.h.temp
-	$(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
+	$(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h

 sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
@@ -1029,6 +1035,9 @@ fts5.c: $(FTS5_SRC)
 fts5.lo:	fts5.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c fts5.c

+sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c
+

 # Rules to build the 'testfixture' application.
 #
@@ -1070,7 +1079,7 @@ fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
 	./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA)

 valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
-	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M $(FUZZDATA)
+	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)

 # Minimal testing that runs in less than 3 minutes
 #
@@ -1080,7 +1089,7 @@ quicktest:	./testfixture$(TEXE)
 # This is the common case.  Run many tests that do not take too long,
 # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
 #
-test:	$(TESTPROGS) fastfuzztest
+test:	$(TESTPROGS) sourcetest fastfuzztest
 	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)

 # Run a test using valgrind.  This can take a really long time
@@ -1102,7 +1111,7 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
 	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
 	echo "static const char *tclsh_main_loop(void){" >> $@
 	echo "static const char *zMainloop = " >> $@
-	$(NAWK) -f $(TOP)/tool/tostr.awk $(TOP)/tool/spaceanal.tcl >> $@
+	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
 	echo "; return zMainloop; }" >> $@

 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
@@ -1132,6 +1141,12 @@ wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.c
 speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)

+rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
+	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
+
+loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
+	$(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS)
+
 # This target will fail if the SQLite amalgamation contains any exported
 # symbols that do not begin with "sqlite3_". It is run as part of the
 # releasetest.tcl script.
@@ -1157,7 +1172,7 @@ THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
                  $(TOP)/test/tt3_lookaside1.c

 threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC)
-	$(LTLINK) $(TOP)/test/threadtest3.c sqlite3.lo -o $@ $(TLIBS)
+	$(LTLINK) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.lo -o $@ $(TLIBS)

 threadtest: threadtest3$(TEXE)
 	./threadtest3$(TEXE)
@@ -1171,9 +1186,9 @@ lib_install:	libsqlite3.la
 	$(INSTALL) -d $(DESTDIR)$(libdir)
 	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
 	
-install:	sqlite3$(BEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
+install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
 	$(INSTALL) -d $(DESTDIR)$(bindir)
-	$(LTINSTALL) sqlite3$(BEXE) $(DESTDIR)$(bindir)
+	$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
 	$(INSTALL) -d $(DESTDIR)$(includedir)
 	$(INSTALL) -m 0644 sqlite3.h $(DESTDIR)$(includedir)
 	$(INSTALL) -m 0644 $(TOP)/src/sqlite3ext.h $(DESTDIR)$(includedir)
@@ -1209,6 +1224,7 @@ clean:
 	rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
 	rm -f sqlite-*-output.vsix
 	rm -f mptester mptester.exe
+	rm -f rbu rbu.exe
 	rm -f fuzzershell fuzzershell.exe
 	rm -f fuzzcheck fuzzcheck.exe
 	rm -f sqldiff sqldiff.exe
@@ -118,11 +118,6 @@ READLINE_FLAGS =
 LIBREADLINE =
 #LIBREADLINE = -static -lreadline -ltermcap

-#### Which "awk" program provides nawk compatibilty
-#
-# NAWK = nawk
-NAWK = awk
-
 # You should not have to change anything below this line
 ###############################################################################
 include $(TOP)/main.mk
@@ -10,11 +10,13 @@
 #
 TOP = .

+# <<mark>>
 # Set this non-0 to create and use the SQLite amalgamation file.
 #
 !IFNDEF USE_AMALGAMATION
 USE_AMALGAMATION = 1
 !ENDIF
+# <</mark>>

 # Set this non-0 to enable full warnings (-W4, etc) when compiling.
 #
@@ -50,7 +52,7 @@ API_ARMOR = 0
 !IFNDEF NO_WARN
 !IF $(USE_FULLWARN)!=0
 NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4130 -wd4152 -wd4189 -wd4206
-NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706
+NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4305 -wd4306 -wd4702 -wd4706
 !ENDIF
 !ENDIF

@@ -68,11 +70,13 @@ USE_WP81_OPTS = 0
 SPLIT_AMALGAMATION = 0
 !ENDIF

+# <<mark>>
 # Set this non-0 to use the International Components for Unicode (ICU).
 #
 !IFNDEF USE_ICU
 USE_ICU = 0
 !ENDIF
+# <</mark>>

 # Set this non-0 to dynamically link to the MSVC runtime library.
 #
@@ -130,12 +134,20 @@ FOR_WINRT = 0
 FOR_UAP = 0
 !ENDIF

+# Set this non-0 to compile binaries suitable for the Windows 10 platform.
+#
+!IFNDEF FOR_WIN10
+FOR_WIN10 = 0
+!ENDIF
+
+# <<mark>>
 # Set this non-0 to skip attempting to look for and/or link with the Tcl
 # runtime library.
 #
 !IFNDEF NO_TCL
 NO_TCL = 0
 !ENDIF
+# <</mark>>

 # Set this to non-0 to create and use PDBs.
 #
@@ -186,6 +198,49 @@ DEBUG = 0
 OPTIMIZATIONS = 2
 !ENDIF

+# Set the source code file to be used by executables and libraries when
+# they need the amalgamation.
+#
+!IFNDEF SQLITE3C
+!IF $(SPLIT_AMALGAMATION)!=0
+SQLITE3C = sqlite3-all.c
+!ELSE
+SQLITE3C = sqlite3.c
+!ENDIF
+!ENDIF
+
+# Set the include code file to be used by executables and libraries when
+# they need SQLite.
+#
+!IFNDEF SQLITE3H
+SQLITE3H = sqlite3.h
+!ENDIF
+
+# This is the name to use for the SQLite dynamic link library (DLL).
+#
+!IFNDEF SQLITE3DLL
+SQLITE3DLL = sqlite3.dll
+!ENDIF
+
+# This is the name to use for the SQLite import library (LIB).
+#
+!IFNDEF SQLITE3LIB
+SQLITE3LIB = sqlite3.lib
+!ENDIF
+
+# This is the name to use for the SQLite shell executable (EXE).
+#
+!IFNDEF SQLITE3EXE
+SQLITE3EXE = sqlite3.exe
+!ENDIF
+
+# This is the argument used to set the program database (PDB) file for the
+# SQLite shell executable (EXE).
+#
+!IFNDEF SQLITE3EXEPDB
+SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
+!ENDIF
+
 # These are the "standard" SQLite compilation options used when compiling for
 # the Windows platform.
 #
@@ -195,6 +250,19 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF

+# These are the "extended" SQLite compilation options used when compiling for
+# the Windows 10 platform.
+#
+!IFNDEF EXT_FEATURE_FLAGS
+!IF $(FOR_WIN10)!=0
+EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1
+EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_SYSTEM_MALLOC=1
+EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_OMIT_LOCALTIME=1
+!ELSE
+EXT_FEATURE_FLAGS =
+!ENDIF
+!ENDIF
+
 ###############################################################################
 ############################### END OF OPTIONS ################################
 ###############################################################################
@@ -292,9 +360,9 @@ NSDKLIBPATH = $(NSDKLIBPATH:\\=\)
 # will run on the platform that is doing the build.
 #
 !IF $(USE_FULLWARN)!=0
-BCC = $(NCC) -nologo -W4
+BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS)
 !ELSE
-BCC = $(NCC) -nologo -W3
+BCC = $(NCC) -nologo -W3 $(CCOPTS) $(BCCOPTS)
 !ENDIF

 # Check if assembly code listings should be generated for the source
@@ -322,13 +390,22 @@ NLTLIBPATHS = $(NLTLIBPATHS) "/LIBPATH:$(NUCRTLIBPATH)"
 # same unless your are cross-compiling.)
 #
 !IF $(USE_FULLWARN)!=0
-TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1
+TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
 !ELSE
-TCC = $(CC) -nologo -W3
+TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
 !ENDIF

-TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise
-RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src
+TCC = $(TCC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src -fp:precise
+RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
+
+# Adjust the names of the primary targets for use with Windows 10.
+#
+!IF $(FOR_WIN10)!=0
+SQLITE3DLL = winsqlite3.dll
+SQLITE3LIB = winsqlite3.lib
+SQLITE3EXE = winsqlite3shell.exe
+SQLITE3EXEPDB =
+!ENDIF

 # Check if we want to use the "stdcall" calling convention when compiling.
 # This is not supported by the compilers for non-x86 platforms.  It should
@@ -337,7 +414,7 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src
 # to how the Tcl library functions are declared and exported (i.e. without
 # an explicit calling convention, which results in "cdecl").
 #
-!IF $(USE_STDCALL)!=0
+!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
 !IF "$(PLATFORM)"=="x86"
 CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
 SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
@@ -358,7 +435,7 @@ SHELL_CCONV_OPTS =
 # These are additional compiler options used for the core library.
 #
 !IFNDEF CORE_COMPILE_OPTS
-!IF $(DYNAMIC_SHELL)!=0
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport)
 !ELSE
 CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS)
@@ -369,7 +446,7 @@ CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS)
 # when linking.
 #
 !IFNDEF CORE_LINK_DEP
-!IF $(DYNAMIC_SHELL)!=0
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 CORE_LINK_DEP =
 !ELSE
 CORE_LINK_DEP = sqlite3.def
@@ -379,7 +456,7 @@ CORE_LINK_DEP = sqlite3.def
 # These are additional linker options used for the core library.
 #
 !IFNDEF CORE_LINK_OPTS
-!IF $(DYNAMIC_SHELL)!=0
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 CORE_LINK_OPTS =
 !ELSE
 CORE_LINK_OPTS = /DEF:sqlite3.def
@@ -389,30 +466,41 @@ CORE_LINK_OPTS = /DEF:sqlite3.def
 # These are additional compiler options used for the shell executable.
 #
 !IFNDEF SHELL_COMPILE_OPTS
-!IF $(DYNAMIC_SHELL)!=0
-SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport)
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport)
 !ELSE
-SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS)
+SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS)
+!ENDIF
+!ENDIF
+
+# This is the source code that the shell executable should be compiled
+# with.
+#
+!IFNDEF SHELL_CORE_SRC
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+SHELL_CORE_SRC =
+!ELSE
+SHELL_CORE_SRC = $(SQLITE3C)
 !ENDIF
 !ENDIF

 # This is the core library that the shell executable should depend on.
 #
 !IFNDEF SHELL_CORE_DEP
-!IF $(DYNAMIC_SHELL)!=0
-SHELL_CORE_DEP = sqlite3.dll
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+SHELL_CORE_DEP = $(SQLITE3DLL)
 !ELSE
-SHELL_CORE_DEP = libsqlite3.lib
+SHELL_CORE_DEP =
 !ENDIF
 !ENDIF

 # This is the core library that the shell executable should link with.
 #
 !IFNDEF SHELL_CORE_LIB
-!IF $(DYNAMIC_SHELL)!=0
-SHELL_CORE_LIB = sqlite3.lib
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+SHELL_CORE_LIB = $(SQLITE3LIB)
 !ELSE
-SHELL_CORE_LIB = libsqlite3.lib
+SHELL_CORE_LIB =
 !ENDIF
 !ENDIF

@@ -441,12 +529,19 @@ TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
 RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
 !ENDIF

+# C compiler options for the Windows 10 platform (needs MSVC 2015).
+#
+!IF $(FOR_WIN10)!=0
+TCC = $(TCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
+BCC = $(BCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
+!ENDIF
+
 # Also, we need to dynamically link to the correct MSVC runtime
 # when compiling for WinRT (e.g. debug or release) OR if the
 # USE_CRT_DLL option is set to force dynamically linking to the
 # MSVC runtime library.
 #
-!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0
+!IF $(FOR_WINRT)!=0 || $(FOR_WIN10)!=0 || $(USE_CRT_DLL)!=0
 !IF $(DEBUG)>1
 TCC = $(TCC) -MDd
 BCC = $(BCC) -MDd
@@ -464,6 +559,7 @@ BCC = $(BCC) -MT
 !ENDIF
 !ENDIF

+# <<mark>>
 # The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in
 # any extension header files by default.  For non-amalgamation
 # builds, we need to make sure the compiler can find these.
@@ -487,6 +583,7 @@ MKSQLITE3C_ARGS = --linemacros
 MKSQLITE3C_ARGS =
 !ENDIF
 !ENDIF
+# <</mark>>

 # Define -DNDEBUG to compile without debugging (i.e., for production usage)
 # Omitting the define will cause extra debugging code to be inserted and
@@ -498,7 +595,7 @@ BCC = $(BCC) -DNDEBUG
 RCC = $(RCC) -DNDEBUG
 !ENDIF

-!IF $(DEBUG)>0 || $(API_ARMOR)!=0
+!IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0
 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1
 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
 !ENDIF
@@ -551,6 +648,7 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
 !ENDIF
 !ENDIF

+# <<mark>>
 # The locations of the Tcl header and library files.  Also, the library that
 # non-stubs enabled programs using Tcl must link against.  These variables
 # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment
@@ -569,6 +667,14 @@ TCLLIBDIR = c:\tcl\lib
 LIBTCL = tcl85.lib
 !ENDIF

+!IFNDEF LIBTCLSTUB
+LIBTCLSTUB = tclstub85.lib
+!ENDIF
+
+!IFNDEF LIBTCLPATH
+LIBTCLPATH = c:\tcl\bin
+!ENDIF
+
 # The locations of the ICU header and library files.  These variables
 # (ICUINCDIR, ICULIBDIR, and LIBICU) may be overridden via the environment
 # prior to running nmake in order to match the actual installed location on
@@ -594,6 +700,7 @@ LIBICU = icuuc.lib icuin.lib
 !IFNDEF TCLSH_CMD
 TCLSH_CMD = tclsh85
 !ENDIF
+# <</mark>>

 # Compiler options needed for programs that use the readline() library.
 #
@@ -651,8 +758,8 @@ REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1
 # Add the required and optional SQLite compilation options into the command
 # lines used to invoke the MSVC code and resource compilers.
 #
-TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS)
-RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS)
+TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
+RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)

 # Add in any optional parameters specified on the commane line, e.g.
 # nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1"
@@ -692,6 +799,7 @@ TCC = $(TCC) -Zi
 BCC = $(BCC) -Zi
 !ENDIF

+# <<mark>>
 # If ICU support is enabled, add the compiler options for it.
 #
 !IF $(USE_ICU)!=0
@@ -702,6 +810,7 @@ RCC = $(RCC) -I$(TOP)\ext\icu
 TCC = $(TCC) -I$(ICUINCDIR)
 RCC = $(RCC) -I$(ICUINCDIR)
 !ENDIF
+# <</mark>>

 # Command line prefixes for compiling code, compiling resources,
 # linking, etc.
@@ -781,7 +890,7 @@ LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib

 # When compiling for UAP, some extra linker options are also required.
 #
-!IF $(FOR_UAP)!=0
+!IF $(FOR_UAP)!=0 || $(FOR_WIN10)!=0
 LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib
 LTLINKOPTS = $(LTLINKOPTS) mincore.lib
 !IFDEF PSDKLIBPATH
@@ -792,9 +901,12 @@ LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)"
 # If either debugging or symbols are enabled, enable PDBs.
 #
 !IF $(DEBUG)>1 || $(SYMBOLS)!=0
-LDFLAGS = /DEBUG
+LDFLAGS = /DEBUG $(LDOPTS)
+!ELSE
+LDFLAGS = $(LDOPTS)
 !ENDIF

+# <<mark>>
 # Start with the Tcl related linker options.
 #
 !IF $(NO_TCL)==0
@@ -808,16 +920,12 @@ LTLIBS = $(LIBTCL)
 LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR)
 LTLIBS = $(LTLIBS) $(LIBICU)
 !ENDIF
-
-# nawk compatible awk.
-#
-!IFNDEF NAWK
-NAWK = gawk.exe
-!ENDIF
+# <</mark>>

 # You should not have to change anything below this line
 ###############################################################################

+# <<mark>>
 # Object files for the SQLite library (non-amalgamation).
 #
 LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
@@ -835,12 +943,13 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
         mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
         notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
         pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
-         random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
+         random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
         table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
         update.lo util.lo vacuum.lo \
         vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
         vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
         utf.lo vtab.lo
+# <</mark>>

 # Object files for the amalgamation.
 #
@@ -848,11 +957,15 @@ LIBOBJS1 = sqlite3.lo

 # Determine the real value of LIBOBJ based on the 'configure' script
 #
+# <<mark>>
 !IF $(USE_AMALGAMATION)==0
 LIBOBJ = $(LIBOBJS0)
 !ELSE
+# <</mark>>
 LIBOBJ = $(LIBOBJS1)
+# <<mark>>
 !ENDIF
+# <</mark>>

 # Determine if embedded resource compilation and usage are enabled.
 #
@@ -862,6 +975,7 @@ LIBRESOBJS = sqlite3res.lo
 LIBRESOBJS =
 !ENDIF

+# <<mark>>
 # All of the source code files.
 #
 SRC1 = \
@@ -1020,7 +1134,7 @@ SRC5 = \
  opcodes.h \
  parse.c \
  parse.h \
-  sqlite3.h
+  $(SQLITE3H)

 # All source code files.
 #
@@ -1067,6 +1181,7 @@ TESTSRC = \
  $(TOP)\src\test_tclvar.c \
  $(TOP)\src\test_thread.c \
  $(TOP)\src\test_vfs.c \
+  $(TOP)\src\test_windirent.c \
  $(TOP)\src\test_wsd.c \
  $(TOP)\ext\fts3\fts3_term.c \
  $(TOP)\ext\fts3\fts3_test.c \
@@ -1080,11 +1195,10 @@ TESTEXT = \
  $(TOP)\ext\misc\eval.c \
  $(TOP)\ext\misc\fileio.c \
  $(TOP)\ext\misc\fuzzer.c \
-  fts5.c \
  $(TOP)\ext\fts5\fts5_tcl.c \
  $(TOP)\ext\fts5\fts5_test_mi.c \
+  $(TOP)\ext\fts5\fts5_test_tok.c \
  $(TOP)\ext\misc\ieee754.c \
-  $(TOP)\ext\misc\json1.c \
  $(TOP)\ext\misc\nextchar.c \
  $(TOP)\ext\misc\percentile.c \
  $(TOP)\ext\misc\regexp.c \
@@ -1164,7 +1278,7 @@ HDR = \
   $(TOP)\src\pcache.h \
   parse.h \
   $(TOP)\src\pragma.h \
-   sqlite3.h \
+   $(SQLITE3H) \
   $(TOP)\src\sqlite3ext.h \
   $(TOP)\src\sqliteInt.h \
   $(TOP)\src\sqliteLimit.h \
@@ -1199,7 +1313,7 @@ EXTHDR = $(EXTHDR) \
 #
 TESTPROGS = \
  testfixture.exe \
-  sqlite3.exe \
+  $(SQLITE3EXE) \
  sqlite3_analyzer.exe \
  sqldiff.exe

@@ -1210,45 +1324,89 @@ FUZZDATA = \
  $(TOP)\test\fuzzdata2.db \
  $(TOP)\test\fuzzdata3.db \
  $(TOP)\test\fuzzdata4.db
+# <</mark>>

-# Extra compiler options for various shell tools
+# Additional compiler options for the shell.  These are only effective
+# when the shell is not being dynamically linked.
 #
-SHELL_COMPILE_OPTS = -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
-FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
+!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
+!ENDIF

-# Standard options to testfixture
+# <<mark>>
+# Extra compiler options for various test tools.
+#
+MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
+FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
+FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
+
+# Standard options to testfixture.
 #
 TESTOPTS = --verbose=file --output=test-out.txt

+# Extra targets for the "all" target that require Tcl.
+#
+!IF $(NO_TCL)==0
+ALL_TCL_TARGETS = libtclsqlite3.lib
+!ELSE
+ALL_TCL_TARGETS =
+!ENDIF
+# <</mark>>
+
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-all:	dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib
+all:	dll libsqlite3.lib shell $(ALL_TCL_TARGETS)
+
+# Dynamic link library section.
+#
+dll: $(SQLITE3DLL)
+
+# Shell executable.
+#
+shell: $(SQLITE3EXE)

 libsqlite3.lib:	$(LIBOBJ)
 	$(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS)

+# <<mark>>
 libtclsqlite3.lib:	tclsqlite.lo libsqlite3.lib
-	$(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS)
+	$(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS)
+# <</mark>>

-sqlite3.exe:	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c \
-		/link /pdb:sqlite3sh.pdb $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
+$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
+	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)

-sqldiff.exe:	$(TOP)\tool\sqldiff.c sqlite3.c sqlite3.h
-	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c sqlite3.c
+# <<mark>>
+sqlite3.def: libsqlite3.lib
+	echo EXPORTS > sqlite3.def
+	dumpbin /all libsqlite3.lib \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_.*)$$" \1 \
+		| sort >> sqlite3.def
+# <</mark>>

-fuzzershell.exe:	$(TOP)\tool\fuzzershell.c sqlite3.c sqlite3.h
-	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) \
-	  $(TOP)\tool\fuzzershell.c sqlite3.c
+$(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
+		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)

-fuzzcheck.exe:	$(TOP)\test\fuzzcheck.c sqlite3.c sqlite3.h
-	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c sqlite3.c
+# <<mark>>
+sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-mptester.exe:	$(TOP)\mptest\mptest.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h
-	$(LTLINK) $(NO_WARN) $(SHELL_COMPILE_OPTS) $(TOP)\mptest\mptest.c \
-		/link $(LTLINKOPTS) $(LTLIBPATHS) $(SHELL_LINK_OPTS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
+srcck1.exe:	$(TOP)\tool\srcck1.c
+	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
+
+sourcetest:	srcck1.exe sqlite3.c
+	srcck1.exe sqlite3.c
+
+fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
+fuzzcheck.exe:	$(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
+mptester.exe:	$(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

 MPTEST1 = mptester mptest.db $(TOP)\mptest\crash01.test --repeat 20
 MPTEST2 = mptester mptest.db $(TOP)\mptest\multiwrite01.test --repeat 20
@@ -1291,29 +1449,22 @@ sqlite3.c:	.target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl

 sqlite3-all.c:	sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
 	$(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl
-
-# Set the source code file to be used by executables and libraries when
-# they need the amalgamation.
-#
-!IF $(SPLIT_AMALGAMATION)!=0
-SQLITE3C = sqlite3-all.c
-!ELSE
-SQLITE3C = sqlite3.c
-!ENDIF
+# <</mark>>

 # Rule to build the amalgamation
 #
 sqlite3.lo:	$(SQLITE3C)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C)

+# <<mark>>
 # Rules to build the LEMON compiler generator
 #
-lempar.c:	$(TOP)\src\lempar.c
-	copy $(TOP)\src\lempar.c .
+lempar.c:	$(TOP)\tool\lempar.c
+	copy $(TOP)\tool\lempar.c .

 lemon.exe:	$(TOP)\tool\lemon.c lempar.c
 	$(BCC) $(NO_WARN) -Daccess=_access \
-		-Fe$@ $(TOP)\tool\lemon.c /link $(NLTLINKOPTS) $(NLTLIBPATHS)
+		-Fe$@ $(TOP)\tool\lemon.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)

 # Rules to build individual *.lo files from generated *.c files. This
 # applies to:
@@ -1326,20 +1477,24 @@ parse.lo:	parse.c $(HDR)

 opcodes.lo:	opcodes.c
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c opcodes.c
+# <</mark>>

 # Rule to build the Win32 resources object file.
 #
 !IF $(USE_RC)!=0
-$(LIBRESOBJS):	$(TOP)\src\sqlite3.rc $(HDR)
+# <<block1>>
+$(LIBRESOBJS):	$(TOP)\src\sqlite3.rc $(SQLITE3H)
 	echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h
 	for /F %%V in ('type "$(TOP)\VERSION"') do ( \
 		echo #define SQLITE_RESOURCE_VERSION %%V \
-			| $(NAWK) "/.*/ { gsub(/[.]/,\",\");print }" >> sqlite3rc.h \
+			| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact . ^, >> sqlite3rc.h \
 	)
 	echo #endif >> sqlite3rc.h
 	$(LTRCOMPILE) -fo $(LIBRESOBJS) $(TOP)\src\sqlite3.rc
+# <</block1>>
 !ENDIF

+# <<mark>>
 # Rules to build individual *.lo files from files in the src directory.
 #
 alter.lo:	$(TOP)\src\alter.c $(HDR)
@@ -1570,37 +1725,37 @@ tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR)
 tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR)
 	$(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c

-tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(LIBRESOBJS)
-	$(LTLINK) $(SQLITE3C) /link $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
+	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)

 # Rules to build opcodes.c and opcodes.h
 #
-opcodes.c:	opcodes.h $(TOP)\mkopcodec.awk
-	$(NAWK) -f $(TOP)\mkopcodec.awk opcodes.h > opcodes.c
+opcodes.c:	opcodes.h $(TOP)\tool\mkopcodec.tcl
+	$(TCLSH_CMD) $(TOP)\tool\mkopcodec.tcl opcodes.h > opcodes.c

-opcodes.h:	parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk
-	type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk > opcodes.h
+opcodes.h:	parse.h $(TOP)\src\vdbe.c $(TOP)\tool\mkopcodeh.tcl
+	type parse.h $(TOP)\src\vdbe.c | $(TCLSH_CMD) $(TOP)\tool\mkopcodeh.tcl > opcodes.h

 # Rules to build parse.c and parse.h - the outputs of lemon.
 #
 parse.h:	parse.c

-parse.c:	$(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk
+parse.c:	$(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl
 	del /Q parse.y parse.h parse.h.temp 2>NUL
 	copy $(TOP)\src\parse.y .
-	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
+	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) parse.y
 	move parse.h parse.h.temp
-	$(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp > parse.h
+	$(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h

-sqlite3.h:	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
-	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > sqlite3.h
+$(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
+	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H)

 sqlite3ext.h: .target_source
 	copy tsrc\sqlite3ext.h .

 mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
-	$(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) \
-		$(TOP)\tool\mkkeywordhash.c /link $(NLTLINKOPTS) $(NLTLIBPATHS)
+	$(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \
+		$(TOP)\tool\mkkeywordhash.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)

 keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 	.\mkkeywordhash.exe > keywordhash.h
@@ -1694,7 +1849,7 @@ FTS5_SRC = \
 fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
 	copy $(TOP)\ext\fts5\fts5parse.y .
 	del /Q fts5parse.h 2>NUL
-	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) fts5parse.y
+	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) fts5parse.y

 fts5parse.h: fts5parse.c

@@ -1711,6 +1866,9 @@ fts5_ext.lo:	fts5.c $(HDR) $(EXTHDR)
 fts5.dll:	fts5_ext.lo
 	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo

+sqlite3rbu.lo:	$(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rbu\sqlite3rbu.c
+
 # Rules to build the 'testfixture' application.
 #
 # If using the amalgamation, use sqlite3.c directly to build the test
@@ -1730,25 +1888,30 @@ TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1)
 !ENDIF

-testfixture.exe:	$(TESTFIXTURE_SRC) $(LIBRESOBJS) $(HDR)
+testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
 		-DBUILD_sqlite -I$(TCLINCDIR) \
 		$(TESTFIXTURE_SRC) \
-		/link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)

 extensiontest: testfixture.exe testloadext.dll
+	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)

 fulltest:	$(TESTPROGS) fuzztest
+	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\all.test $(TESTOPTS)

 soaktest:	$(TESTPROGS)
+	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\all.test -soak=1 $(TESTOPTS)

 fulltestonly:	$(TESTPROGS) fuzztest
+	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\full.test

-queryplantest:	testfixture.exe sqlite3.exe
+queryplantest:	testfixture.exe shell
+	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\permutations.test queryplanner $(TESTOPTS)

 fuzztest:	fuzzcheck.exe
@@ -1759,30 +1922,33 @@ fastfuzztest:	fuzzcheck.exe

 # Minimal testing that runs in less than 3 minutes (on a fast machine)
 #
-quicktest:	testfixture.exe
+quicktest:	testfixture.exe sourcetest
+	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS)

 # This is the common case.  Run many tests that do not take too long,
 # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
 #
-test:	$(TESTPROGS) fastfuzztest
+test:	$(TESTPROGS) sourcetest fastfuzztest
+	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS)

 smoketest:	$(TESTPROGS)
+	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)

-sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
+sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
 	echo #define TCLSH 2 > $@
 	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
 	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
 	echo static const char *tclsh_main_loop(void){ >> $@
 	echo static const char *zMainloop = >> $@
-	$(NAWK) -f $(TOP)\tool\tostr.awk $(TOP)\tool\spaceanal.tcl >> $@
+	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@
 	echo ; return zMainloop; } >> $@

 sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
-		/link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)

 testloadext.lo:	$(TOP)\src\test_loadext.c
 	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
@@ -1790,45 +1956,51 @@ testloadext.lo:	$(TOP)\src\test_loadext.c
 testloadext.dll: testloadext.lo
 	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo

-showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C)
+showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
-		$(TOP)\tool\showdb.c $(SQLITE3C)
+		$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C)
+showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
-		$(TOP)\tool\showstat4.c $(SQLITE3C)
+		$(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-showjournal.exe:	$(TOP)\tool\showjournal.c $(SQLITE3C)
+showjournal.exe:	$(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
-		$(TOP)\tool\showjournal.c $(SQLITE3C)
+		$(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C)
+showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
-		$(TOP)\tool\showwal.c $(SQLITE3C)
+		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C)
+fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
-		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C)
+		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C)
+rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
-		$(TOP)\tool\rollback-test.c $(SQLITE3C)
+		$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-LogEst.exe:	$(TOP)\tool\logest.c sqlite3.h
-	$(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c
+LogEst.exe:	$(TOP)\tool\logest.c $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)

-wordcount.exe:	$(TOP)\test\wordcount.c $(SQLITE3C)
+wordcount.exe:	$(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
-		$(TOP)\test\wordcount.c $(SQLITE3C)
+		$(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C)
+speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
-		$(TOP)\test\speedtest1.c $(SQLITE3C)
+		$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
+rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \
+		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+# <</mark>>

 clean:
 	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
-	del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL
-	del /Q sqlite3.h opcodes.c opcodes.h 2>NUL
+	del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
+# <<mark>>
+	del /Q $(SQLITE3C) $(SQLITE3H) opcodes.c opcodes.h 2>NUL
 	del /Q lemon.* lempar.c parse.* 2>NUL
 	del /Q mkkeywordhash.* keywordhash.h 2>NUL
 	del /Q notasharedlib.* 2>NUL
@@ -1844,8 +2016,8 @@ clean:
 	del /Q testfixture.exe test.db 2>NUL
 	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL
 	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
-	del /Q mptester.exe wordcount.exe 2>NUL
-	del /Q sqlite3.exe sqlite3.dll sqlite3.def 2>NUL
+	del /Q mptester.exe wordcount.exe rbu.exe 2>NUL
+	del /Q $(SQLITE3EXE) $(SQLITE3DLL) sqlite3.def 2>NUL
 	del /Q sqlite3.c sqlite3-*.c 2>NUL
 	del /Q sqlite3rc.h 2>NUL
 	del /Q shell.c sqlite3ext.h 2>NUL
@@ -1853,16 +2025,4 @@ clean:
 	del /Q sqlite-*-output.vsix 2>NUL
 	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
 	del /Q fts5.* fts5parse.* 2>NUL
-
-# Dynamic link library section.
-#
-dll: sqlite3.dll
-
-sqlite3.def: libsqlite3.lib
-	echo EXPORTS > sqlite3.def
-	dumpbin /all libsqlite3.lib \
-		| $(NAWK) "/ 1 _?sqlite3_/ { sub(/^.* _?/,\"\");print }" \
-		| sort >> sqlite3.def
-
-sqlite3.dll: $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
-	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+# <</mark>>
@@ -1,673 +0,0 @@
-#!/usr/make
-#
-# Makefile for SQLITE on VxWorks
-
-ifeq ($(FORCPU),)
-  FORCPU = SH32gnule
-endif
-
-TOOL_FAMILY = gnu
-
-include $(WIND_USR)/tool/gnu/make.$(FORCPU)
-
-#### The toplevel directory of the source tree.  This is the directory
-#    that contains this "Makefile.in" and the "configure.in" script.
-#
-TOP = .
-
-#### C Compiler and options for use in building executables that
-#    will run on the platform that is doing the build.
-#
-BCC = gcc -g -O2
-#BCC = /opt/ancic/bin/c89 -0
-
-#### If the target operating system supports the "usleep()" system
-#    call, then define the HAVE_USLEEP macro for all C modules.
-#
-USLEEP = 
-#USLEEP = -DHAVE_USLEEP=1
-
-#### If you want the SQLite library to be safe for use within a 
-#    multi-threaded program, then define the following macro
-#    appropriately:
-#
-THREADSAFE = -DSQLITE_THREADSAFE=1
-#THREADSAFE = -DSQLITE_THREADSAFE=0
-
-#### Specify any extra linker options needed to make the library
-#    thread safe
-#
-#THREADLIB = -lpthread
-THREADLIB = 
-
-#### Specify any extra libraries needed to access required functions.
-#
-ifeq ($(CPU),SH32) 
-  # for SH4 shared library
-  TLIBS_SHARED += -L$(WIND_USR)/lib/sh/SH32/commonle/PIC
-else 
-  # for all other CPUs shared library
-  TLIBS_SHARED += $(LD_LINK_PATH_ATEND) $(LD_PARTIAL_LAST_FLAGS)
-endif
-# for static library
-TLIBS += $(LD_LINK_PATH_ATEND) $(LD_PARTIAL_LAST_FLAGS)
-
-#### Leave SQLITE_DEBUG undefined for maximum speed.  Use SQLITE_DEBUG=1
-#    to check for memory leaks.  Use SQLITE_DEBUG=2 to print a log of all
-#    malloc()s and free()s in order to track down memory leaks.
-#    
-#    SQLite uses some expensive assert() statements in the inner loop.
-#    You can make the library go almost twice as fast if you compile
-#    with -DNDEBUG=1
-#
-#OPTS = -DSQLITE_DEBUG=2
-#OPTS = -DSQLITE_DEBUG=1
-#OPTS = 
-OPTS = -DNDEBUG=1 -DSQLITE_OS_UNIX=1 $(THREADSAFE)
-OPTS += -DSQLITE_OMIT_LOAD_EXTENSION=1
-OPTS += -DSQLITE_ENABLE_LOCKING_STYLE=1
-OPTS += -DSQLITE_THREAD_OVERRIDE_LOCK=0
-OPTS += -DSQLITE_ENABLE_COLUMN_METADATA=1
-OPTS += -DHAVE_FDATASYNC=1
-
-#### The suffix to add to executable files.  ".exe" for windows.
-#    Nothing for unix.
-#
-EXE = .vxe
-#EXE =
-
-#### C Compile and options for use in building executables that 
-#    will run on the target platform.  This is usually the same
-#    as BCC, unless you are cross-compiling.
-#
-#TCC = gcc -O6
-#TCC = gcc -g -O0 -Wall
-#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage
-#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6
-TCC = $(CC) $(DEFINE_CC) -O2 -g -mrtp $(CC_ARCH_SPEC) -D_REENTRANT=1 -D_VX_CPU=_VX_$(CPU) -D_VX_TOOL_FAMILY=$(TOOL_FAMILY) -D_VX_TOOL=$(TOOL)
-TCC += -I$(WIND_USR)/h -I$(WIND_USR)/h/wrn/coreip
-#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive
-
-#TCC_SHARED = $(TCC) -fPIC
-TCC_SHARED = $(TCC)
-
-#### Tools used to build a static library.
-#
-#ARX = ar cr
-#ARX = /opt/mingw/bin/i386-mingw32-ar cr
-AR += cr
-#RANLIB = ranlib
-#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib
-
-#MKSHLIB = gcc -shared
-#SO = so
-#SHPREFIX = lib
-MKSHLIB = $(CC) $(DEFINE_CC) -mrtp -shared $(CC_ARCH_SPEC) -D_VX_CPU=_VX_$(CPU) -D_VX_TOOL_FAMILY=$(TOOL_FAMILY) -D_VX_TOOL=$(TOOL)
-SO = so
-SHPREFIX = lib
-
-#### Extra compiler options needed for programs that use the TCL library.
-#
-#TCL_FLAGS =
-#TCL_FLAGS = -DSTATIC_BUILD=1
-TCL_FLAGS = -I/home/drh/tcltk/8.5linux
-#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1
-#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux
-
-#### Linker options needed to link against the TCL library.
-#
-#LIBTCL = -ltcl -lm -ldl
-LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl
-#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt
-#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc
-
-#### Additional objects for SQLite library when TCL support is enabled.
-TCLOBJ =
-#TCLOBJ = tclsqlite.o
-
-#### Compiler options needed for programs that use the readline() library.
-#
-READLINE_FLAGS =
-#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline
-
-#### Linker options needed by programs using readline() must link against.
-#
-LIBREADLINE =
-#LIBREADLINE = -static -lreadline -ltermcap
-
-#### Which "awk" program provides nawk compatibilty
-#
-# NAWK = nawk
-NAWK = awk
-
-
-#### Pasted and adapted main.mk file
-###############################################################################
-# The following macros should be defined before this script is
-# invoked:
-#
-# TOP              The toplevel directory of the source tree.  This is the
-#                  directory that contains this "Makefile.in" and the
-#                  "configure.in" script.
-#
-# BCC              C Compiler and options for use in building executables that
-#                  will run on the platform that is doing the build.
-#
-# THREADLIB        Specify any extra linker options needed to make the library
-#                  thread safe
-#
-# OPTS             Extra compiler command-line options.
-#
-# EXE              The suffix to add to executable files.  ".exe" for windows
-#                  and "" for Unix.
-#
-# TCC              C Compiler and options for use in building executables that 
-#                  will run on the target platform.  This is usually the same
-#                  as BCC, unless you are cross-compiling.
-#
-# AR               Tools used to build a static library.
-# RANLIB
-#
-# TCL_FLAGS        Extra compiler options needed for programs that use the
-#                  TCL library.
-#
-# LIBTCL           Linker options needed to link against the TCL library.
-#
-# READLINE_FLAGS   Compiler options needed for programs that use the
-#                  readline() library.
-#
-# LIBREADLINE      Linker options needed by programs using readline() must
-#                  link against.
-#
-# NAWK             Nawk compatible awk program.  Older (obsolete?) solaris
-#                  systems need this to avoid using the original AT&T AWK.
-#
-# Once the macros above are defined, the rest of this make script will
-# build the SQLite library and testing tools.
-################################################################################
-
-# This is how we compile
-#
-TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP)
-TCCX_SHARED = $(TCC_SHARED) $(OPTS) -I. -I$(TOP)/src -I$(TOP) \
-	-I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 \
-	-I$(TOP)/ext/async
-
-# Object files for the SQLite library.
-#
-LIBOBJ+= alter.o analyze.o attach.o auth.o \
-         backup.o bitvec.o btmutex.o btree.o build.o \
-         callback.o complete.o date.o delete.o expr.o fault.o \
-         fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
-         fts3_tokenizer.o fts3_tokenizer1.o \
-         func.o global.o hash.o \
-         icu.o insert.o journal.o legacy.o loadext.o \
-         main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \
-         memjournal.o \
-         mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \
-         notify.o opcodes.o os.o os_unix.o os_win.o \
-         pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \
-         random.o resolve.o rowset.o rtree.o select.o status.o \
-         table.o tokenize.o trigger.o \
-         update.o util.o vacuum.o \
-         vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o \
-         walker.o where.o utf.o vtab.o
-
-
-
-# All of the source code files.
-#
-SRC = \
-  $(TOP)/src/alter.c \
-  $(TOP)/src/analyze.c \
-  $(TOP)/src/attach.c \
-  $(TOP)/src/auth.c \
-  $(TOP)/src/backup.c \
-  $(TOP)/src/bitvec.c \
-  $(TOP)/src/btmutex.c \
-  $(TOP)/src/btree.c \
-  $(TOP)/src/btree.h \
-  $(TOP)/src/btreeInt.h \
-  $(TOP)/src/build.c \
-  $(TOP)/src/callback.c \
-  $(TOP)/src/complete.c \
-  $(TOP)/src/ctime.c \
-  $(TOP)/src/date.c \
-  $(TOP)/src/delete.c \
-  $(TOP)/src/expr.c \
-  $(TOP)/src/fault.c \
-  $(TOP)/src/func.c \
-  $(TOP)/src/global.c \
-  $(TOP)/src/hash.c \
-  $(TOP)/src/hash.h \
-  $(TOP)/src/hwtime.h \
-  $(TOP)/src/insert.c \
-  $(TOP)/src/journal.c \
-  $(TOP)/src/legacy.c \
-  $(TOP)/src/loadext.c \
-  $(TOP)/src/main.c \
-  $(TOP)/src/malloc.c \
-  $(TOP)/src/mem0.c \
-  $(TOP)/src/mem1.c \
-  $(TOP)/src/mem2.c \
-  $(TOP)/src/mem3.c \
-  $(TOP)/src/mem5.c \
-  $(TOP)/src/memjournal.c \
-  $(TOP)/src/msvc.h \
-  $(TOP)/src/mutex.c \
-  $(TOP)/src/mutex.h \
-  $(TOP)/src/mutex_noop.c \
-  $(TOP)/src/mutex_unix.c \
-  $(TOP)/src/mutex_w32.c \
-  $(TOP)/src/notify.c \
-  $(TOP)/src/os.c \
-  $(TOP)/src/os.h \
-  $(TOP)/src/os_common.h \
-  $(TOP)/src/os_setup.h \
-  $(TOP)/src/os_unix.c \
-  $(TOP)/src/os_win.c \
-  $(TOP)/src/os_win.h \
-  $(TOP)/src/pager.c \
-  $(TOP)/src/pager.h \
-  $(TOP)/src/parse.y \
-  $(TOP)/src/pcache.c \
-  $(TOP)/src/pcache.h \
-  $(TOP)/src/pcache1.c \
-  $(TOP)/src/pragma.c \
-  $(TOP)/src/prepare.c \
-  $(TOP)/src/printf.c \
-  $(TOP)/src/random.c \
-  $(TOP)/src/resolve.c \
-  $(TOP)/src/rowset.c \
-  $(TOP)/src/select.c \
-  $(TOP)/src/status.c \
-  $(TOP)/src/shell.c \
-  $(TOP)/src/sqlite.h.in \
-  $(TOP)/src/sqlite3ext.h \
-  $(TOP)/src/sqliteInt.h \
-  $(TOP)/src/sqliteLimit.h \
-  $(TOP)/src/table.c \
-  $(TOP)/src/tclsqlite.c \
-  $(TOP)/src/tokenize.c \
-  $(TOP)/src/trigger.c \
-  $(TOP)/src/utf.c \
-  $(TOP)/src/update.c \
-  $(TOP)/src/util.c \
-  $(TOP)/src/vacuum.c \
-  $(TOP)/src/vdbe.c \
-  $(TOP)/src/vdbe.h \
-  $(TOP)/src/vdbeapi.c \
-  $(TOP)/src/vdbeaux.c \
-  $(TOP)/src/vdbeblob.c \
-  $(TOP)/src/vdbemem.c \
-  $(TOP)/src/vdbeInt.h \
-  $(TOP)/src/vtab.c \
-  $(TOP)/src/walker.c \
-  $(TOP)/src/where.c
-
-# Source code for extensions
-#
-SRC += \
-  $(TOP)/ext/fts1/fts1.c \
-  $(TOP)/ext/fts1/fts1.h \
-  $(TOP)/ext/fts1/fts1_hash.c \
-  $(TOP)/ext/fts1/fts1_hash.h \
-  $(TOP)/ext/fts1/fts1_porter.c \
-  $(TOP)/ext/fts1/fts1_tokenizer.h \
-  $(TOP)/ext/fts1/fts1_tokenizer1.c
-SRC += \
-  $(TOP)/ext/fts2/fts2.c \
-  $(TOP)/ext/fts2/fts2.h \
-  $(TOP)/ext/fts2/fts2_hash.c \
-  $(TOP)/ext/fts2/fts2_hash.h \
-  $(TOP)/ext/fts2/fts2_icu.c \
-  $(TOP)/ext/fts2/fts2_porter.c \
-  $(TOP)/ext/fts2/fts2_tokenizer.h \
-  $(TOP)/ext/fts2/fts2_tokenizer.c \
-  $(TOP)/ext/fts2/fts2_tokenizer1.c
-SRC += \
-  $(TOP)/ext/fts3/fts3.c \
-  $(TOP)/ext/fts3/fts3.h \
-  $(TOP)/ext/fts3/fts3_expr.c \
-  $(TOP)/ext/fts3/fts3_expr.h \
-  $(TOP)/ext/fts3/fts3_hash.c \
-  $(TOP)/ext/fts3/fts3_hash.h \
-  $(TOP)/ext/fts3/fts3_icu.c \
-  $(TOP)/ext/fts3/fts3_porter.c \
-  $(TOP)/ext/fts3/fts3_tokenizer.h \
-  $(TOP)/ext/fts3/fts3_tokenizer.c \
-  $(TOP)/ext/fts3/fts3_tokenizer1.c
-SRC += \
-  $(TOP)/ext/icu/sqliteicu.h \
-  $(TOP)/ext/icu/icu.c 
-SRC += \
-  $(TOP)/ext/rtree/rtree.h \
-  $(TOP)/ext/rtree/rtree.c
-
-
-# Generated source code files
-#
-SRC += \
-  keywordhash.h \
-  opcodes.c \
-  opcodes.h \
-  parse.c \
-  parse.h \
-  sqlite3.h
-
-
-# Source code to the test files.
-#
-TESTSRC = \
-  $(TOP)/src/test1.c \
-  $(TOP)/src/test2.c \
-  $(TOP)/src/test3.c \
-  $(TOP)/src/test4.c \
-  $(TOP)/src/test5.c \
-  $(TOP)/src/test6.c \
-  $(TOP)/src/test7.c \
-  $(TOP)/src/test8.c \
-  $(TOP)/src/test9.c \
-  $(TOP)/src/test_autoext.c \
-  $(TOP)/src/test_async.c \
-  $(TOP)/src/test_backup.c \
-  $(TOP)/src/test_btree.c \
-  $(TOP)/src/test_config.c \
-  $(TOP)/src/test_devsym.c \
-  $(TOP)/src/test_func.c \
-  $(TOP)/src/test_hexio.c \
-  $(TOP)/src/test_journal.c \
-  $(TOP)/src/test_malloc.c \
-  $(TOP)/src/test_md5.c \
-  $(TOP)/src/test_mutex.c \
-  $(TOP)/src/test_onefile.c \
-  $(TOP)/src/test_osinst.c \
-  $(TOP)/src/test_pcache.c \
-  $(TOP)/src/test_schema.c \
-  $(TOP)/src/test_server.c \
-  $(TOP)/src/test_tclvar.c \
-  $(TOP)/src/test_thread.c \
-  $(TOP)/src/test_vfs.c \
-  $(TOP)/src/test_wsd.c \
-
-#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
-#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c
-
-TESTSRC2 = \
-  $(TOP)/src/attach.c $(TOP)/src/backup.c $(TOP)/src/btree.c                   \
-  $(TOP)/src/build.c $(TOP)/src/ctime.c $(TOP)/src/date.c                      \
-  $(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/os.c      \
-  $(TOP)/src/os_unix.c $(TOP)/src/os_win.c                 \
-  $(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c                  \
-  $(TOP)/src/printf.c $(TOP)/src/random.c $(TOP)/src/pcache.c                  \
-  $(TOP)/src/pcache1.c $(TOP)/src/select.c $(TOP)/src/tokenize.c               \
-  $(TOP)/src/utf.c $(TOP)/src/util.c $(TOP)/src/vdbeapi.c $(TOP)/src/vdbeaux.c \
-  $(TOP)/src/vdbe.c $(TOP)/src/vdbemem.c $(TOP)/src/where.c parse.c            \
-  $(TOP)/ext/fts3/fts3.c $(TOP)/ext/fts3/fts3_expr.c                           \
-  $(TOP)/ext/fts3/fts3_tokenizer.c                                             \
-  $(TOP)/ext/async/sqlite3async.c
-
-# Header files used by all library source files.
-#
-HDR = \
-   $(TOP)/src/btree.h \
-   $(TOP)/src/btreeInt.h \
-   $(TOP)/src/hash.h \
-   $(TOP)/src/hwtime.h \
-   keywordhash.h \
-   $(TOP)/src/msvc.h \
-   $(TOP)/src/mutex.h \
-   opcodes.h \
-   $(TOP)/src/os.h \
-   $(TOP)/src/os_common.h \
-   $(TOP)/src/os_setup.h \
-   $(TOP)/src/os_win.h \
-   $(TOP)/src/pager.h \
-   $(TOP)/src/pcache.h \
-   parse.h  \
-   sqlite3.h  \
-   $(TOP)/src/sqlite3ext.h \
-   $(TOP)/src/sqliteInt.h  \
-   $(TOP)/src/sqliteLimit.h \
-   $(TOP)/src/vdbe.h \
-   $(TOP)/src/vdbeInt.h
-
-# Header files used by extensions
-#
-EXTHDR += \
-  $(TOP)/ext/fts1/fts1.h \
-  $(TOP)/ext/fts1/fts1_hash.h \
-  $(TOP)/ext/fts1/fts1_tokenizer.h
-EXTHDR += \
-  $(TOP)/ext/fts2/fts2.h \
-  $(TOP)/ext/fts2/fts2_hash.h \
-  $(TOP)/ext/fts2/fts2_tokenizer.h
-EXTHDR += \
-  $(TOP)/ext/fts3/fts3.h \
-  $(TOP)/ext/fts3/fts3_expr.h \
-  $(TOP)/ext/fts3/fts3_hash.h \
-  $(TOP)/ext/fts3/fts3_tokenizer.h
-EXTHDR += \
-  $(TOP)/ext/rtree/rtree.h
-EXTHDR += \
-  $(TOP)/ext/icu/sqliteicu.h
-
-# This is the default Makefile target.  The objects listed here
-# are what get build when you type just "make" with no arguments.
-#
-all:	sqlite3.h libsqlite3.a sqlite3$(EXE)
-
-libsqlite3.a:	$(LIBOBJ)
-	$(AR) libsqlite3.a $(LIBOBJ)
-	$(RANLIB) libsqlite3.a
-
-$(SHPREFIX)sqlite3.$(SO):	$(LIBOBJ)
-	$(MKSHLIB) -o $(SHPREFIX)sqlite3.$(SO) $(LIBOBJ) $(TLIBS_SHARED)
-
-sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h
-	$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE)      \
-		$(TOP)/src/shell.c                      \
-		$(LIBREADLINE) $(TLIBS) $(THREADLIB) -L. -lsqlite3
-
-# This target creates a directory named "tsrc" and fills it with
-# copies of all of the C source code and header files needed to
-# build on the target system.  Some of the C source code and header
-# files are automatically generated.  This target takes care of
-# all that automatic generation.
-#
-target_source:	$(SRC)
-	rm -rf tsrc
-	mkdir tsrc
-	cp -f $(SRC) tsrc
-	rm tsrc/sqlite.h.in tsrc/parse.y
-	touch target_source
-
-sqlite3.c:	target_source $(TOP)/tool/mksqlite3c.tcl
-	tclsh $(TOP)/tool/mksqlite3c.tcl
-	cp sqlite3.c tclsqlite3.c
-	cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
-
-fts2amal.c:	target_source $(TOP)/ext/fts2/mkfts2amal.tcl
-	tclsh $(TOP)/ext/fts2/mkfts2amal.tcl
-
-fts3amal.c:	target_source $(TOP)/ext/fts3/mkfts3amal.tcl
-	tclsh $(TOP)/ext/fts3/mkfts3amal.tcl
-
-# Rules to build the LEMON compiler generator
-#
-lemon:	$(TOP)/tool/lemon.c $(TOP)/src/lempar.c
-	$(BCC) -o lemon $(TOP)/tool/lemon.c
-	cp $(TOP)/src/lempar.c .
-
-# Rules to build individual *.o files from generated *.c files. This
-# applies to:
-#
-#     parse.o
-#     opcodes.o
-#
-%.o: %.c $(HDR)
-	$(TCCX_SHARED) -c $<
-
-# Rules to build individual *.o files from files in the src directory.
-#
-%.o: $(TOP)/src/%.c $(HDR)
-	$(TCCX_SHARED) -c $<
-
-tclsqlite.o:	$(TOP)/src/tclsqlite.c $(HDR)
-	$(TCCX_SHARED) $(TCL_FLAGS) -c $(TOP)/src/tclsqlite.c
-
-
-
-# Rules to build opcodes.c and opcodes.h
-#
-opcodes.c:	opcodes.h $(TOP)/mkopcodec.awk
-	$(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c
-
-opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
-	cat parse.h $(TOP)/src/vdbe.c | \
-		$(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h
-
-# Rules to build parse.c and parse.h - the outputs of lemon.
-#
-parse.h:	parse.c
-
-parse.c:	$(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk
-	cp $(TOP)/src/parse.y .
-	rm -f parse.h
-	./lemon $(OPTS) parse.y
-	mv parse.h parse.h.temp
-	awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
-
-sqlite3.h:	$(TOP)/src/sqlite.h.in 
-	sed -e s/--VERS--/`cat ${TOP}/VERSION`/ \
-	    -e s/--VERSION-NUMBER--/`cat ${TOP}/VERSION | sed 's/[^0-9]/ /g' | $(NAWK) '{printf "%d%03d%03d",$$1,$$2,$$3}'`/ \
-                 $(TOP)/src/sqlite.h.in >sqlite3.h
-
-keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
-	$(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c
-	./mkkeywordhash >keywordhash.h
-
-
-
-# Rules to build the extension objects.
-#
-icu.o:	$(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c
-
-fts2.o:	$(TOP)/ext/fts2/fts2.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2.c
-
-fts2_hash.o:	$(TOP)/ext/fts2/fts2_hash.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_hash.c
-
-fts2_icu.o:	$(TOP)/ext/fts2/fts2_icu.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_icu.c
-
-fts2_porter.o:	$(TOP)/ext/fts2/fts2_porter.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_porter.c
-
-fts2_tokenizer.o:	$(TOP)/ext/fts2/fts2_tokenizer.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c
-
-fts2_tokenizer1.o:	$(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c
-
-fts3.o:	$(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c
-
-fts3_expr.o:	$(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c
-
-fts3_hash.o:	$(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c
-
-fts3_icu.o:	$(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c
-
-fts3_porter.o:	$(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c
-
-fts3_tokenizer.o:	$(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c
-
-fts3_tokenizer1.o:	$(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c
-
-rtree.o:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
-	$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
-
-
-# Rules for building test programs and for running tests
-#
-tclsqlite3:	$(TOP)/src/tclsqlite.c libsqlite3.a
-	$(TCCX_SHARED) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \
-		$(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB)
-
-
-# Rules to build the 'testfixture' application.
-#
-TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
-TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
-
-testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c
-	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                            \
-		$(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c                \
-		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB) libsqlite3.a
-
-amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c
-	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                            \
-		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c                  \
-		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
-
-fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
-	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                            \
-	-DSQLITE_ENABLE_FTS3=1                                               \
-		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
-		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
-
-fulltest:	testfixture$(EXE) sqlite3$(EXE)
-	./testfixture$(EXE) $(TOP)/test/all.test
-
-soaktest:	testfixture$(EXE) sqlite3$(EXE)
-	./testfixture$(EXE) $(TOP)/test/all.test -soak=1
-
-fulltestonly:	testfixture$(EXE) sqlite3$(EXE)
-	./testfixture$(EXE) $(TOP)/test/full.test
-
-test:	testfixture$(EXE) sqlite3$(EXE)
-	./testfixture$(EXE) $(TOP)/test/veryquick.test
-
-sqlite3_analyzer$(EXE):	$(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \
-			$(TOP)/tool/spaceanal.tcl
-	sed \
-	  -e '/^#/d' \
-	  -e 's,\\,\\\\,g' \
-	  -e 's,",\\",g' \
-	  -e 's,^,",' \
-	  -e 's,$$,\\n",' \
-	  $(TOP)/tool/spaceanal.tcl >spaceanal_tcl.h
-	$(TCCX) $(TCL_FLAGS)                  $(TESTFIXTURE_FLAGS)                                 \
-		-DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_DEBUG=1 -DSQLITE_PRIVATE="" \
-		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c                    \
-		-o sqlite3_analyzer$(EXE)                                      \
-		$(LIBTCL) $(THREADLIB)
-
-TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
-$(TEST_EXTENSION): $(TOP)/src/test_loadext.c
-	$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
-
-extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
-	./testfixture$(EXE) $(TOP)/test/loadext.test
-
-clean:	
-	rm -f *.o sqlite3$(EXE) libsqlite3.a sqlite3.h opcodes.*
-	rm -f lemon lempar.c parse.* sqlite*.tar.gz mkkeywordhash keywordhash.h
-	rm -f $(PUBLISH)
-	rm -f *.da *.bb *.bbg gmon.out
-	rm -rf quota2a quota2b quota2c
-	rm -rf tsrc target_source
-	rm -f testloadext.dll libtestloadext.so
-	rm -f sqlite3.c fts?amal.c tclsqlite3.c
-	rm -f sqlite3rc.h
-	rm -f shell.c sqlite3ext.h
-	rm -f $(SHPREFIX)sqlite3.$(SO)
@@ -1 +1 @@
-3.9.0
+3.11.0
@@ -1,34 +0,0 @@
-#!/usr/bin/awk
-#
-# This script appends additional token codes to the end of the
-# parse.h file that lemon generates.  These extra token codes are
-# not used by the parser.  But they are used by the tokenizer and/or
-# the code generator.
-#
-#
-BEGIN {
-  max = 0
-}
-/^#define TK_/ {
-  print $0
-  if( max<$3 ) max = $3
-}
-END {
-  printf "#define TK_%-29s %4d\n", "TO_TEXT",         ++max
-  printf "#define TK_%-29s %4d\n", "TO_BLOB",         ++max
-  printf "#define TK_%-29s %4d\n", "TO_NUMERIC",      ++max
-  printf "#define TK_%-29s %4d\n", "TO_INT",          ++max
-  printf "#define TK_%-29s %4d\n", "TO_REAL",         ++max
-  printf "#define TK_%-29s %4d\n", "ISNOT",           ++max
-  printf "#define TK_%-29s %4d\n", "END_OF_FILE",     ++max
-  printf "#define TK_%-29s %4d\n", "ILLEGAL",         ++max
-  printf "#define TK_%-29s %4d\n", "SPACE",           ++max
-  printf "#define TK_%-29s %4d\n", "UNCLOSED_STRING", ++max
-  printf "#define TK_%-29s %4d\n", "FUNCTION",        ++max
-  printf "#define TK_%-29s %4d\n", "COLUMN",          ++max
-  printf "#define TK_%-29s %4d\n", "AGG_FUNCTION",    ++max
-  printf "#define TK_%-29s %4d\n", "AGG_COLUMN",      ++max
-  printf "#define TK_%-29s %4d\n", "UMINUS",          ++max
-  printf "#define TK_%-29s %4d\n", "UPLUS",           ++max
-  printf "#define TK_%-29s %4d\n", "REGISTER",        ++max
-}
@@ -6,13 +6,15 @@ libsqlite3_la_SOURCES = sqlite3.c
 libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8

 bin_PROGRAMS = sqlite3
-sqlite3_SOURCES = shell.c sqlite3.c sqlite3.h
-sqlite3_LDADD = @READLINE_LIBS@
+sqlite3_SOURCES = shell.c sqlite3.h
+EXTRA_sqlite3_SOURCES = sqlite3.c
+sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
+sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
 sqlite3_CFLAGS = $(AM_CFLAGS)

 include_HEADERS = sqlite3.h sqlite3ext.h

-EXTRA_DIST = sqlite3.1 tea
+EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt
 pkgconfigdir = ${libdir}/pkgconfig
 pkgconfig_DATA = sqlite3.pc

@@ -0,0 +1,892 @@
+#### DO NOT EDIT ####
+# This makefile is automatically generated from the Makefile.msc at
+# the root of the canonical SQLite source tree (not the
+# amalgamation tarball) using the tool/mkmsvcmin.tcl
+# script.
+#
+
+#
+# nmake Makefile for SQLite
+#
+###############################################################################
+############################## START OF OPTIONS ###############################
+###############################################################################
+
+# The toplevel directory of the source tree.  This is the directory
+# that contains this "Makefile.msc".
+#
+TOP = .
+
+
+# Set this non-0 to enable full warnings (-W4, etc) when compiling.
+#
+!IFNDEF USE_FULLWARN
+USE_FULLWARN = 0
+!ENDIF
+
+# Set this non-0 to use "stdcall" calling convention for the core library
+# and shell executable.
+#
+!IFNDEF USE_STDCALL
+USE_STDCALL = 0
+!ENDIF
+
+# Set this non-0 to have the shell executable link against the core dynamic
+# link library.
+#
+!IFNDEF DYNAMIC_SHELL
+DYNAMIC_SHELL = 0
+!ENDIF
+
+# Set this non-0 to enable extra code that attempts to detect misuse of the
+# SQLite API.
+#
+!IFNDEF API_ARMOR
+API_ARMOR = 0
+!ENDIF
+
+# If necessary, create a list of harmless compiler warnings to disable when
+# compiling the various tools.  For the SQLite source code itself, warnings,
+# if any, will be disabled from within it.
+#
+!IFNDEF NO_WARN
+!IF $(USE_FULLWARN)!=0
+NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4130 -wd4152 -wd4189 -wd4206
+NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4305 -wd4306 -wd4702 -wd4706
+!ENDIF
+!ENDIF
+
+# Set this non-0 to use the library paths and other options necessary for
+# Windows Phone 8.1.
+#
+!IFNDEF USE_WP81_OPTS
+USE_WP81_OPTS = 0
+!ENDIF
+
+# Set this non-0 to split the SQLite amalgamation file into chunks to
+# be used for debugging with Visual Studio.
+#
+!IFNDEF SPLIT_AMALGAMATION
+SPLIT_AMALGAMATION = 0
+!ENDIF
+
+
+# Set this non-0 to dynamically link to the MSVC runtime library.
+#
+!IFNDEF USE_CRT_DLL
+USE_CRT_DLL = 0
+!ENDIF
+
+# Set this non-0 to link to the RPCRT4 library.
+#
+!IFNDEF USE_RPCRT4_LIB
+USE_RPCRT4_LIB = 0
+!ENDIF
+
+# Set this non-0 to generate assembly code listings for the source code
+# files.
+#
+!IFNDEF USE_LISTINGS
+USE_LISTINGS = 0
+!ENDIF
+
+# Set this non-0 to attempt setting the native compiler automatically
+# for cross-compiling the command line tools needed during the compilation
+# process.
+#
+!IFNDEF XCOMPILE
+XCOMPILE = 0
+!ENDIF
+
+# Set this non-0 to use the native libraries paths for cross-compiling
+# the command line tools needed during the compilation process.
+#
+!IFNDEF USE_NATIVE_LIBPATHS
+USE_NATIVE_LIBPATHS = 0
+!ENDIF
+
+# Set this 0 to skip the compiling and embedding of version resources.
+#
+!IFNDEF USE_RC
+USE_RC = 1
+!ENDIF
+
+# Set this non-0 to compile binaries suitable for the WinRT environment.
+# This setting does not apply to any binaries that require Tcl to operate
+# properly (i.e. the text fixture, etc).
+#
+!IFNDEF FOR_WINRT
+FOR_WINRT = 0
+!ENDIF
+
+# Set this non-0 to compile binaries suitable for the UAP environment.
+# This setting does not apply to any binaries that require Tcl to operate
+# properly (i.e. the text fixture, etc).
+#
+!IFNDEF FOR_UAP
+FOR_UAP = 0
+!ENDIF
+
+# Set this non-0 to compile binaries suitable for the Windows 10 platform.
+#
+!IFNDEF FOR_WIN10
+FOR_WIN10 = 0
+!ENDIF
+
+
+# Set this to non-0 to create and use PDBs.
+#
+!IFNDEF SYMBOLS
+SYMBOLS = 1
+!ENDIF
+
+# Set this to non-0 to use the SQLite debugging heap subsystem.
+#
+!IFNDEF MEMDEBUG
+MEMDEBUG = 0
+!ENDIF
+
+# Set this to non-0 to use the Win32 native heap subsystem.
+#
+!IFNDEF WIN32HEAP
+WIN32HEAP = 0
+!ENDIF
+
+# Set this to non-0 to enable OSTRACE() macros, which can be useful when
+# debugging.
+#
+!IFNDEF OSTRACE
+OSTRACE = 0
+!ENDIF
+
+# Set this to one of the following values to enable various debugging
+# features.  Each level includes the debugging options from the previous
+# levels.  Currently, the recognized values for DEBUG are:
+#
+# 0 == NDEBUG: Disables assert() and other runtime diagnostics.
+# 1 == SQLITE_ENABLE_API_ARMOR: extra attempts to detect misuse of the API.
+# 2 == Disables NDEBUG and all optimizations and then enables PDBs.
+# 3 == SQLITE_DEBUG: Enables various diagnostics messages and code.
+# 4 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call.
+# 5 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros.
+# 6 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros.
+#
+!IFNDEF DEBUG
+DEBUG = 0
+!ENDIF
+
+# Enable use of available compiler optimizations?  Normally, this should be
+# non-zero.  Setting this to zero, thus disabling all compiler optimizations,
+# can be useful for testing.
+#
+!IFNDEF OPTIMIZATIONS
+OPTIMIZATIONS = 2
+!ENDIF
+
+# Set the source code file to be used by executables and libraries when
+# they need the amalgamation.
+#
+!IFNDEF SQLITE3C
+!IF $(SPLIT_AMALGAMATION)!=0
+SQLITE3C = sqlite3-all.c
+!ELSE
+SQLITE3C = sqlite3.c
+!ENDIF
+!ENDIF
+
+# Set the include code file to be used by executables and libraries when
+# they need SQLite.
+#
+!IFNDEF SQLITE3H
+SQLITE3H = sqlite3.h
+!ENDIF
+
+# This is the name to use for the SQLite dynamic link library (DLL).
+#
+!IFNDEF SQLITE3DLL
+SQLITE3DLL = sqlite3.dll
+!ENDIF
+
+# This is the name to use for the SQLite import library (LIB).
+#
+!IFNDEF SQLITE3LIB
+SQLITE3LIB = sqlite3.lib
+!ENDIF
+
+# This is the name to use for the SQLite shell executable (EXE).
+#
+!IFNDEF SQLITE3EXE
+SQLITE3EXE = sqlite3.exe
+!ENDIF
+
+# This is the argument used to set the program database (PDB) file for the
+# SQLite shell executable (EXE).
+#
+!IFNDEF SQLITE3EXEPDB
+SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
+!ENDIF
+
+# These are the "standard" SQLite compilation options used when compiling for
+# the Windows platform.
+#
+!IFNDEF OPT_FEATURE_FLAGS
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
+!ENDIF
+
+# These are the "extended" SQLite compilation options used when compiling for
+# the Windows 10 platform.
+#
+!IFNDEF EXT_FEATURE_FLAGS
+!IF $(FOR_WIN10)!=0
+EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1
+EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_SYSTEM_MALLOC=1
+EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_OMIT_LOCALTIME=1
+!ELSE
+EXT_FEATURE_FLAGS =
+!ENDIF
+!ENDIF
+
+###############################################################################
+############################### END OF OPTIONS ################################
+###############################################################################
+
+# This assumes that MSVC is always installed in 32-bit Program Files directory
+# and sets the variable for use in locating other 32-bit installs accordingly.
+#
+PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\..
+PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\)
+
+# Check for the predefined command macro CC.  This should point to the compiler
+# binary for the target platform.  If it is not defined, simply define it to
+# the legacy default value 'cl.exe'.
+#
+!IFNDEF CC
+CC = cl.exe
+!ENDIF
+
+# Check for the command macro LD.  This should point to the linker binary for
+# the target platform.  If it is not defined, simply define it to the legacy
+# default value 'link.exe'.
+#
+!IFNDEF LD
+LD = link.exe
+!ENDIF
+
+# Check for the predefined command macro RC.  This should point to the resource
+# compiler binary for the target platform.  If it is not defined, simply define
+# it to the legacy default value 'rc.exe'.
+#
+!IFNDEF RC
+RC = rc.exe
+!ENDIF
+
+# Check for the MSVC runtime library path macro.  Othertise, this value will
+# default to the 'lib' directory underneath the MSVC installation directory.
+#
+!IFNDEF CRTLIBPATH
+CRTLIBPATH = $(VCINSTALLDIR)\lib
+!ENDIF
+
+CRTLIBPATH = $(CRTLIBPATH:\\=\)
+
+# Check for the command macro NCC.  This should point to the compiler binary
+# for the platform the compilation process is taking place on.  If it is not
+# defined, simply define it to have the same value as the CC macro.  When
+# cross-compiling, it is suggested that this macro be modified via the command
+# line (since nmake itself does not provide a built-in method to guess it).
+# For example, to use the x86 compiler when cross-compiling for x64, a command
+# line similar to the following could be used (all on one line):
+#
+#     nmake /f Makefile.msc sqlite3.dll
+#           XCOMPILE=1 USE_NATIVE_LIBPATHS=1
+#
+# Alternatively, the full path and file name to the compiler binary for the
+# platform the compilation process is taking place may be specified (all on
+# one line):
+#
+#     nmake /f Makefile.msc sqlite3.dll
+#           "NCC=""%VCINSTALLDIR%\bin\cl.exe"""
+#           USE_NATIVE_LIBPATHS=1
+#
+!IFDEF NCC
+NCC = $(NCC:\\=\)
+!ELSEIF $(XCOMPILE)!=0
+NCC = "$(VCINSTALLDIR)\bin\$(CC)"
+NCC = $(NCC:\\=\)
+!ELSE
+NCC = $(CC)
+!ENDIF
+
+# Check for the MSVC native runtime library path macro.  Othertise,
+# this value will default to the 'lib' directory underneath the MSVC
+# installation directory.
+#
+!IFNDEF NCRTLIBPATH
+NCRTLIBPATH = $(VCINSTALLDIR)\lib
+!ENDIF
+
+NCRTLIBPATH = $(NCRTLIBPATH:\\=\)
+
+# Check for the Platform SDK library path macro.  Othertise, this
+# value will default to the 'lib' directory underneath the Windows
+# SDK installation directory (the environment variable used appears
+# to be available when using Visual C++ 2008 or later via the
+# command line).
+#
+!IFNDEF NSDKLIBPATH
+NSDKLIBPATH = $(WINDOWSSDKDIR)\lib
+!ENDIF
+
+NSDKLIBPATH = $(NSDKLIBPATH:\\=\)
+
+# C compiler and options for use in building executables that
+# will run on the platform that is doing the build.
+#
+!IF $(USE_FULLWARN)!=0
+BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS)
+!ELSE
+BCC = $(NCC) -nologo -W3 $(CCOPTS) $(BCCOPTS)
+!ENDIF
+
+# Check if assembly code listings should be generated for the source
+# code files to be compiled.
+#
+!IF $(USE_LISTINGS)!=0
+BCC = $(BCC) -FAcs
+!ENDIF
+
+# Check if the native library paths should be used when compiling
+# the command line tools used during the compilation process.  If
+# so, set the necessary macro now.
+#
+!IF $(USE_NATIVE_LIBPATHS)!=0
+NLTLIBPATHS = "/LIBPATH:$(NCRTLIBPATH)" "/LIBPATH:$(NSDKLIBPATH)"
+
+!IFDEF NUCRTLIBPATH
+NUCRTLIBPATH = $(NUCRTLIBPATH:\\=\)
+NLTLIBPATHS = $(NLTLIBPATHS) "/LIBPATH:$(NUCRTLIBPATH)"
+!ENDIF
+!ENDIF
+
+# C compiler and options for use in building executables that
+# will run on the target platform.  (BCC and TCC are usually the
+# same unless your are cross-compiling.)
+#
+!IF $(USE_FULLWARN)!=0
+TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
+!ELSE
+TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
+!ENDIF
+
+TCC = $(TCC) -DSQLITE_OS_WIN=1 -I$(TOP) -fp:precise
+RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) $(RCOPTS) $(RCCOPTS)
+
+# Adjust the names of the primary targets for use with Windows 10.
+#
+!IF $(FOR_WIN10)!=0
+SQLITE3DLL = winsqlite3.dll
+SQLITE3LIB = winsqlite3.lib
+SQLITE3EXE = winsqlite3shell.exe
+SQLITE3EXEPDB =
+!ENDIF
+
+# Check if we want to use the "stdcall" calling convention when compiling.
+# This is not supported by the compilers for non-x86 platforms.  It should
+# also be noted here that building any target with these "stdcall" options
+# will most likely fail if the Tcl library is also required.  This is due
+# to how the Tcl library functions are declared and exported (i.e. without
+# an explicit calling convention, which results in "cdecl").
+#
+!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
+!IF "$(PLATFORM)"=="x86"
+CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
+SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
+!ELSE
+!IFNDEF PLATFORM
+CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
+SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
+!ELSE
+CORE_CCONV_OPTS =
+SHELL_CCONV_OPTS =
+!ENDIF
+!ENDIF
+!ELSE
+CORE_CCONV_OPTS =
+SHELL_CCONV_OPTS =
+!ENDIF
+
+# These are additional compiler options used for the core library.
+#
+!IFNDEF CORE_COMPILE_OPTS
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport)
+!ELSE
+CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS)
+!ENDIF
+!ENDIF
+
+# These are the additional targets that the core library should depend on
+# when linking.
+#
+!IFNDEF CORE_LINK_DEP
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+CORE_LINK_DEP =
+!ELSE
+CORE_LINK_DEP =
+!ENDIF
+!ENDIF
+
+# These are additional linker options used for the core library.
+#
+!IFNDEF CORE_LINK_OPTS
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+CORE_LINK_OPTS =
+!ELSE
+CORE_LINK_OPTS =
+!ENDIF
+!ENDIF
+
+# These are additional compiler options used for the shell executable.
+#
+!IFNDEF SHELL_COMPILE_OPTS
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport)
+!ELSE
+SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS)
+!ENDIF
+!ENDIF
+
+# This is the source code that the shell executable should be compiled
+# with.
+#
+!IFNDEF SHELL_CORE_SRC
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+SHELL_CORE_SRC =
+!ELSE
+SHELL_CORE_SRC = $(SQLITE3C)
+!ENDIF
+!ENDIF
+
+# This is the core library that the shell executable should depend on.
+#
+!IFNDEF SHELL_CORE_DEP
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+SHELL_CORE_DEP = $(SQLITE3DLL)
+!ELSE
+SHELL_CORE_DEP =
+!ENDIF
+!ENDIF
+
+# This is the core library that the shell executable should link with.
+#
+!IFNDEF SHELL_CORE_LIB
+!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
+SHELL_CORE_LIB = $(SQLITE3LIB)
+!ELSE
+SHELL_CORE_LIB =
+!ENDIF
+!ENDIF
+
+# These are additional linker options used for the shell executable.
+#
+!IFNDEF SHELL_LINK_OPTS
+SHELL_LINK_OPTS = $(SHELL_CORE_LIB)
+!ENDIF
+
+# Check if assembly code listings should be generated for the source
+# code files to be compiled.
+#
+!IF $(USE_LISTINGS)!=0
+TCC = $(TCC) -FAcs
+!ENDIF
+
+# When compiling the library for use in the WinRT environment,
+# the following compile-time options must be used as well to
+# disable use of Win32 APIs that are not available and to enable
+# use of Win32 APIs that are specific to Windows 8 and/or WinRT.
+#
+!IF $(FOR_WINRT)!=0
+TCC = $(TCC) -DSQLITE_OS_WINRT=1
+RCC = $(RCC) -DSQLITE_OS_WINRT=1
+TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
+RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
+!ENDIF
+
+# C compiler options for the Windows 10 platform (needs MSVC 2015).
+#
+!IF $(FOR_WIN10)!=0
+TCC = $(TCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
+BCC = $(BCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
+!ENDIF
+
+# Also, we need to dynamically link to the correct MSVC runtime
+# when compiling for WinRT (e.g. debug or release) OR if the
+# USE_CRT_DLL option is set to force dynamically linking to the
+# MSVC runtime library.
+#
+!IF $(FOR_WINRT)!=0 || $(FOR_WIN10)!=0 || $(USE_CRT_DLL)!=0
+!IF $(DEBUG)>1
+TCC = $(TCC) -MDd
+BCC = $(BCC) -MDd
+!ELSE
+TCC = $(TCC) -MD
+BCC = $(BCC) -MD
+!ENDIF
+!ELSE
+!IF $(DEBUG)>1
+TCC = $(TCC) -MTd
+BCC = $(BCC) -MTd
+!ELSE
+TCC = $(TCC) -MT
+BCC = $(BCC) -MT
+!ENDIF
+!ENDIF
+
+
+# Define -DNDEBUG to compile without debugging (i.e., for production usage)
+# Omitting the define will cause extra debugging code to be inserted and
+# includes extra comments when "EXPLAIN stmt" is used.
+#
+!IF $(DEBUG)==0
+TCC = $(TCC) -DNDEBUG
+BCC = $(BCC) -DNDEBUG
+RCC = $(RCC) -DNDEBUG
+!ENDIF
+
+!IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0
+TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1
+RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
+!ENDIF
+
+!IF $(DEBUG)>2
+TCC = $(TCC) -DSQLITE_DEBUG=1
+RCC = $(RCC) -DSQLITE_DEBUG=1
+!ENDIF
+
+!IF $(DEBUG)>4 || $(OSTRACE)!=0
+TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
+RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
+!ENDIF
+
+!IF $(DEBUG)>5
+TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE=1
+RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE=1
+!ENDIF
+
+# Prevent warnings about "insecure" MSVC runtime library functions
+# being used.
+#
+TCC = $(TCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
+BCC = $(BCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
+RCC = $(RCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
+
+# Prevent warnings about "deprecated" POSIX functions being used.
+#
+TCC = $(TCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
+BCC = $(BCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
+RCC = $(RCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
+
+# Use the SQLite debugging heap subsystem?
+#
+!IF $(MEMDEBUG)!=0
+TCC = $(TCC) -DSQLITE_MEMDEBUG=1
+RCC = $(RCC) -DSQLITE_MEMDEBUG=1
+
+# Use native Win32 heap subsystem instead of malloc/free?
+#
+!ELSEIF $(WIN32HEAP)!=0
+TCC = $(TCC) -DSQLITE_WIN32_MALLOC=1
+RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1
+
+# Validate the heap on every call into the native Win32 heap subsystem?
+#
+!IF $(DEBUG)>3
+TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
+RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
+!ENDIF
+!ENDIF
+
+
+# Compiler options needed for programs that use the readline() library.
+#
+!IFNDEF READLINE_FLAGS
+READLINE_FLAGS = -DHAVE_READLINE=0
+!ENDIF
+
+# The library that programs using readline() must link against.
+#
+!IFNDEF LIBREADLINE
+LIBREADLINE =
+!ENDIF
+
+# Should the database engine be compiled threadsafe
+#
+TCC = $(TCC) -DSQLITE_THREADSAFE=1
+RCC = $(RCC) -DSQLITE_THREADSAFE=1
+
+# Do threads override each others locks by default (1), or do we test (-1)
+#
+TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1
+RCC = $(RCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1
+
+# Any target libraries which libsqlite must be linked against
+#
+!IFNDEF TLIBS
+TLIBS =
+!ENDIF
+
+# Flags controlling use of the in memory btree implementation
+#
+# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to
+# default to file, 2 to default to memory, and 3 to force temporary
+# tables to always be in memory.
+#
+TCC = $(TCC) -DSQLITE_TEMP_STORE=1
+RCC = $(RCC) -DSQLITE_TEMP_STORE=1
+
+# Enable/disable loadable extensions, and other optional features
+# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
+# The same set of OMIT and ENABLE flags should be passed to the
+# LEMON parser generator and the mkkeywordhash tool as well.
+
+# These are the required SQLite compilation options used when compiling for
+# the Windows platform.
+#
+REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100
+
+# If we are linking to the RPCRT4 library, enable features that need it.
+#
+!IF $(USE_RPCRT4_LIB)!=0
+REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1
+!ENDIF
+
+# Add the required and optional SQLite compilation options into the command
+# lines used to invoke the MSVC code and resource compilers.
+#
+TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
+RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
+
+# Add in any optional parameters specified on the commane line, e.g.
+# nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1"
+#
+TCC = $(TCC) $(OPTS)
+RCC = $(RCC) $(OPTS)
+
+# If compiling for debugging, add some defines.
+#
+!IF $(DEBUG)>1
+TCC = $(TCC) -D_DEBUG
+BCC = $(BCC) -D_DEBUG
+RCC = $(RCC) -D_DEBUG
+!ENDIF
+
+# If optimizations are enabled or disabled (either implicitly or
+# explicitly), add the necessary flags.
+#
+!IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
+TCC = $(TCC) -Od
+BCC = $(BCC) -Od
+!ELSEIF $(OPTIMIZATIONS)>=3
+TCC = $(TCC) -Ox
+BCC = $(BCC) -Ox
+!ELSEIF $(OPTIMIZATIONS)==2
+TCC = $(TCC) -O2
+BCC = $(BCC) -O2
+!ELSEIF $(OPTIMIZATIONS)==1
+TCC = $(TCC) -O1
+BCC = $(BCC) -O1
+!ENDIF
+
+# If symbols are enabled (or compiling for debugging), enable PDBs.
+#
+!IF $(DEBUG)>1 || $(SYMBOLS)!=0
+TCC = $(TCC) -Zi
+BCC = $(BCC) -Zi
+!ENDIF
+
+
+# Command line prefixes for compiling code, compiling resources,
+# linking, etc.
+#
+LTCOMPILE = $(TCC) -Fo$@
+LTRCOMPILE = $(RCC) -r
+LTLIB = lib.exe
+LTLINK = $(TCC) -Fe$@
+
+# If requested, link to the RPCRT4 library.
+#
+!IF $(USE_RPCRT4_LIB)!=0
+LTLINK = $(LTLINK) rpcrt4.lib
+!ENDIF
+
+# If a platform was set, force the linker to target that.
+# Note that the vcvars*.bat family of batch files typically
+# set this for you.  Otherwise, the linker will attempt
+# to deduce the binary type based on the object files.
+!IFDEF PLATFORM
+LTLINKOPTS = /NOLOGO /MACHINE:$(PLATFORM)
+LTLIBOPTS = /NOLOGO /MACHINE:$(PLATFORM)
+!ELSE
+LTLINKOPTS = /NOLOGO
+LTLIBOPTS = /NOLOGO
+!ENDIF
+
+# When compiling for use in the WinRT environment, the following
+# linker option must be used to mark the executable as runnable
+# only in the context of an application container.
+#
+!IF $(FOR_WINRT)!=0
+LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER
+!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0"
+!IFNDEF STORELIBPATH
+!IF "$(PLATFORM)"=="x86"
+STORELIBPATH = $(CRTLIBPATH)\store
+!ELSEIF "$(PLATFORM)"=="x64"
+STORELIBPATH = $(CRTLIBPATH)\store\amd64
+!ELSEIF "$(PLATFORM)"=="ARM"
+STORELIBPATH = $(CRTLIBPATH)\store\arm
+!ELSE
+STORELIBPATH = $(CRTLIBPATH)\store
+!ENDIF
+!ENDIF
+STORELIBPATH = $(STORELIBPATH:\\=\)
+LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)"
+!ENDIF
+!ENDIF
+
+# When compiling for Windows Phone 8.1, an extra library path is
+# required.
+#
+!IF $(USE_WP81_OPTS)!=0
+!IFNDEF WP81LIBPATH
+!IF "$(PLATFORM)"=="x86"
+WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86
+!ELSEIF "$(PLATFORM)"=="ARM"
+WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM
+!ELSE
+WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86
+!ENDIF
+!ENDIF
+!ENDIF
+
+# When compiling for Windows Phone 8.1, some extra linker options
+# are also required.
+#
+!IF $(USE_WP81_OPTS)!=0
+!IFDEF WP81LIBPATH
+LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)"
+!ENDIF
+LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE
+LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib
+LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib
+!ENDIF
+
+# When compiling for UAP, some extra linker options are also required.
+#
+!IF $(FOR_UAP)!=0 || $(FOR_WIN10)!=0
+LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib
+LTLINKOPTS = $(LTLINKOPTS) mincore.lib
+!IFDEF PSDKLIBPATH
+LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)"
+!ENDIF
+!ENDIF
+
+# If either debugging or symbols are enabled, enable PDBs.
+#
+!IF $(DEBUG)>1 || $(SYMBOLS)!=0
+LDFLAGS = /DEBUG $(LDOPTS)
+!ELSE
+LDFLAGS = $(LDOPTS)
+!ENDIF
+
+
+# You should not have to change anything below this line
+###############################################################################
+
+
+# Object files for the amalgamation.
+#
+LIBOBJS1 = sqlite3.lo
+
+# Determine the real value of LIBOBJ based on the 'configure' script
+#
+LIBOBJ = $(LIBOBJS1)
+
+# Determine if embedded resource compilation and usage are enabled.
+#
+!IF $(USE_RC)!=0
+LIBRESOBJS = sqlite3res.lo
+!ELSE
+LIBRESOBJS =
+!ENDIF
+
+
+# Additional compiler options for the shell.  These are only effective
+# when the shell is not being dynamically linked.
+#
+!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
+!ENDIF
+
+
+# This is the default Makefile target.  The objects listed here
+# are what get build when you type just "make" with no arguments.
+#
+all:	dll libsqlite3.lib shell
+
+# Dynamic link library section.
+#
+dll: $(SQLITE3DLL)
+
+# Shell executable.
+#
+shell: $(SQLITE3EXE)
+
+libsqlite3.lib:	$(LIBOBJ)
+	$(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS)
+
+
+$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
+	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+
+
+$(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
+		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
+
+
+# Rule to build the amalgamation
+#
+sqlite3.lo:	$(SQLITE3C)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C)
+
+
+# Rule to build the Win32 resources object file.
+#
+!IF $(USE_RC)!=0
+_HASHCHAR=^#
+!IF ![echo !IFNDEF VERSION > rcver.vc] && \
+    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
+    ![echo !ENDIF >> rcver.vc]
+!INCLUDE rcver.vc
+!ENDIF
+
+RESOURCE_VERSION = $(VERSION:^#=)
+RESOURCE_VERSION = $(RESOURCE_VERSION:define=)
+RESOURCE_VERSION = $(RESOURCE_VERSION:SQLITE_VERSION=)
+RESOURCE_VERSION = $(RESOURCE_VERSION:"=)
+RESOURCE_VERSION = $(RESOURCE_VERSION:.=,)
+
+$(LIBRESOBJS):	$(TOP)\sqlite3.rc rcver.vc $(SQLITE3H)
+	echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h
+	echo #define SQLITE_RESOURCE_VERSION $(RESOURCE_VERSION) >> sqlite3rc.h
+	echo #endif >> sqlite3rc.h
+	$(LTRCOMPILE) -fo $(LIBRESOBJS) -DRC_VERONLY $(TOP)\sqlite3.rc
+!ENDIF
+
+
+clean:
+	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
+	del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
@@ -1,32 +0,0 @@
-
-This package contains:
-
- * the SQLite library amalgamation (single file) source code distribution,
- * the shell.c file used to build the sqlite3 shell too, and
- * the sqlite3.h and sqlite3ext.h header files required to link programs
-   and sqlite extensions against the installed libary.
- * autoconf/automake installation infrastucture.
-
-The generic installation instructions for autoconf/automake are found
-in the INSTALL file.
-
-The following SQLite specific boolean options are supported:
-
-  --enable-readline           use readline in shell tool   [default=yes]
-  --enable-threadsafe         build a thread-safe library  [default=yes]
-  --enable-dynamic-extensions support loadable extensions  [default=yes]
-
-The default value for the CFLAGS variable (options passed to the C 
-compiler) includes debugging symbols in the build, resulting in larger
-binaries than are necessary. Override it on the configure command
-line like this:
-
-  $ CFLAGS="-Os" ./configure
-
-to produce a smaller installation footprint.
-
-Other SQLite compilation parameters can also be set using CFLAGS. For
-example:
-
-  $ CFLAGS="-Os -DSQLITE_OMIT_TRIGGERS" ./configure
-
@@ -0,0 +1,113 @@
+This package contains:
+
+ * the SQLite library amalgamation (single file) source code distribution,
+ * the shell.c file used to build the sqlite3 shell too, and
+ * the sqlite3.h and sqlite3ext.h header files required to link programs
+   and sqlite extensions against the installed libary.
+ * autoconf/automake installation infrastucture for building on POSIX
+   compliant systems.
+ * a Makefile.msc and sqlite3.rc for building with Microsoft Visual C++ on
+   Windows.
+
+SUMMARY OF HOW TO BUILD
+=======================
+
+  Unix:      ./configure; make
+  Windows:   nmake /f Makefile.msc
+
+BUILDING ON POSIX
+=================
+
+The generic installation instructions for autoconf/automake are found
+in the INSTALL file.
+
+The following SQLite specific boolean options are supported:
+
+  --enable-readline           use readline in shell tool   [default=yes]
+  --enable-threadsafe         build a thread-safe library  [default=yes]
+  --enable-dynamic-extensions support loadable extensions  [default=yes]
+
+The default value for the CFLAGS variable (options passed to the C
+compiler) includes debugging symbols in the build, resulting in larger
+binaries than are necessary. Override it on the configure command
+line like this:
+
+  $ CFLAGS="-Os" ./configure
+
+to produce a smaller installation footprint.
+
+Other SQLite compilation parameters can also be set using CFLAGS. For
+example:
+
+  $ CFLAGS="-Os -DSQLITE_OMIT_TRIGGERS" ./configure
+
+
+BUILDING WITH MICROSOFT VISUAL C++
+==================================
+
+To compile for Windows using Microsoft Visual C++:
+
+  $ nmake /f Makefile.msc
+
+Using Microsoft Visual C++ 2005 (or later) is recommended.  Several Windows
+platform variants may be built by adding additional macros to the NMAKE
+command line.
+
+Building for WinRT 8.0
+----------------------
+
+  FOR_WINRT=1
+
+Using Microsoft Visual C++ 2012 (or later) is required.  When using the
+above, something like the following macro will need to be added to the
+NMAKE command line as well:
+
+  "NSDKLIBPATH=%WindowsSdkDir%\..\8.0\lib\win8\um\x86"
+
+Building for WinRT 8.1
+----------------------
+
+  FOR_WINRT=1
+
+Using Microsoft Visual C++ 2013 (or later) is required.  When using the
+above, something like the following macro will need to be added to the
+NMAKE command line as well:
+
+  "NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86"
+
+Building for UAP 10.0
+---------------------
+
+  FOR_WINRT=1 FOR_UAP=1
+
+Using Microsoft Visual C++ 2015 (or later) is required.  When using the
+above, something like the following macros will need to be added to the
+NMAKE command line as well:
+
+  "NSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
+  "PSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
+  "NUCRTLIBPATH=%UniversalCRTSdkDir%\..\10\lib\10.0.10586.0\ucrt\x86"
+
+Building for the Windows 10 SDK
+-------------------------------
+
+  FOR_WIN10=1
+
+Using Microsoft Visual C++ 2015 (or later) is required.  When using the
+above, no other macros should be needed on the NMAKE command line.
+
+Other preprocessor defines
+--------------------------
+
+Additionally, preprocessor defines may be specified by using the OPTS macro
+on the NMAKE command line.  However, not all possible preprocessor defines
+may be specified in this manner as some require the amalgamation to be built
+with them enabled (see http://www.sqlite.org/compile.html). For example, the
+following will work:
+
+  "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_ENABLE_JSON1=1"
+
+However, the following will not compile unless the amalgamation was built
+with it enabled:
+
+  "OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1"
@@ -4,11 +4,13 @@
 #
 #   --enable-threadsafe
 #   --enable-readline
+#   --enable-editline
+#   --enable-static-shell
 #   --enable-dynamic-extensions
 #

 AC_PREREQ(2.61)
-AC_INIT(sqlite, 3.7.5, http://www.sqlite.org)
+AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org)
 AC_CONFIG_SRCDIR([sqlite3.c])

 # Use automake.
@@ -18,7 +20,6 @@ AC_SYS_LARGEFILE

 # Check for required programs.
 AC_PROG_CC
-AC_PROG_RANLIB
 AC_PROG_LIBTOOL
 AC_PROG_MKDIR_P

@@ -30,12 +31,29 @@ AC_CONFIG_FILES([Makefile sqlite3.pc])
 AC_SUBST(BUILD_CFLAGS)

 #-----------------------------------------------------------------------
+#   --enable-editline
 #   --enable-readline
 #
+AC_ARG_ENABLE(editline, [AS_HELP_STRING(
+  [--enable-editline], 
+  [use BSD libedit])], 
+  [], [enable_editline=yes])
 AC_ARG_ENABLE(readline, [AS_HELP_STRING(
  [--enable-readline], 
-  [use readline in shell tool (yes, no) [default=yes]])], 
-  [], [enable_readline=yes])
+  [use readline])], 
+  [], [enable_readline=no])
+if test x"$enable_editline" != xno ; then
+  sLIBS=$LIBS
+  LIBS=""
+  AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no])
+  READLINE_LIBS=$LIBS
+  if test x"$LIBS" != "x"; then
+     AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
+  else
+    unset ac_cv_search_readline
+  fi
+  LIBS=$sLIBS
+fi
 if test x"$enable_readline" != xno ; then
  sLIBS=$LIBS
  LIBS=""
@@ -58,6 +76,7 @@ THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
 if test x"$enable_threadsafe" != "xno"; then
  THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
  AC_SEARCH_LIBS(pthread_create, pthread)
+  AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
 fi
 AC_SUBST(THREADSAFE_FLAGS)
 #-----------------------------------------------------------------------
@@ -103,6 +122,21 @@ fi
 AC_SUBST(JSON1_FLAGS)
 #-----------------------------------------------------------------------

+#-----------------------------------------------------------------------
+#   --enable-static-shell
+#
+AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
+  [--enable-static-shell], 
+  [statically link libsqlite3 into shell tool [default=yes]])], 
+  [], [enable_static_shell=yes])
+if test x"$enable_static_shell" == "xyes"; then
+  EXTRA_SHELL_OBJ=sqlite3.$OBJEXT
+else
+  EXTRA_SHELL_OBJ=libsqlite3.la
+fi
+AC_SUBST(EXTRA_SHELL_OBJ)
+#-----------------------------------------------------------------------
+
 AC_CHECK_FUNCS(posix_fallocate)

 #-----------------------------------------------------------------------
@@ -1,708 +0,0 @@
-#! /bin/sh
-# depcomp - compile a program generating dependencies as side-effects
-
-scriptversion=2012-03-27.16; # UTC
-
-# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2006, 2007, 2009, 2010,
-# 2011, 2012 Free Software Foundation, Inc.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# As a special exception to the GNU General Public License, if you
-# distribute this file as part of a program that contains a
-# configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
-
-# Originally written by Alexandre Oliva <oliva@dcc.unicamp.br>.
-
-case $1 in
-  '')
-     echo "$0: No command.  Try '$0 --help' for more information." 1>&2
-     exit 1;
-     ;;
-  -h | --h*)
-    cat <<\EOF
-Usage: depcomp [--help] [--version] PROGRAM [ARGS]
-
-Run PROGRAMS ARGS to compile a file, generating dependencies
-as side-effects.
-
-Environment variables:
-  depmode     Dependency tracking mode.
-  source      Source file read by 'PROGRAMS ARGS'.
-  object      Object file output by 'PROGRAMS ARGS'.
-  DEPDIR      directory where to store dependencies.
-  depfile     Dependency file to output.
-  tmpdepfile  Temporary file to use when outputting dependencies.
-  libtool     Whether libtool is used (yes/no).
-
-Report bugs to <bug-automake@gnu.org>.
-EOF
-    exit $?
-    ;;
-  -v | --v*)
-    echo "depcomp $scriptversion"
-    exit $?
-    ;;
-esac
-
-# A tabulation character.
-tab='	'
-# A newline character.
-nl='
-'
-
-if test -z "$depmode" || test -z "$source" || test -z "$object"; then
-  echo "depcomp: Variables source, object and depmode must be set" 1>&2
-  exit 1
-fi
-
-# Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po.
-depfile=${depfile-`echo "$object" |
-  sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`}
-tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`}
-
-rm -f "$tmpdepfile"
-
-# Some modes work just like other modes, but use different flags.  We
-# parameterize here, but still list the modes in the big case below,
-# to make depend.m4 easier to write.  Note that we *cannot* use a case
-# here, because this file can only contain one case statement.
-if test "$depmode" = hp; then
-  # HP compiler uses -M and no extra arg.
-  gccflag=-M
-  depmode=gcc
-fi
-
-if test "$depmode" = dashXmstdout; then
-   # This is just like dashmstdout with a different argument.
-   dashmflag=-xM
-   depmode=dashmstdout
-fi
-
-cygpath_u="cygpath -u -f -"
-if test "$depmode" = msvcmsys; then
-   # This is just like msvisualcpp but w/o cygpath translation.
-   # Just convert the backslash-escaped backslashes to single forward
-   # slashes to satisfy depend.m4
-   cygpath_u='sed s,\\\\,/,g'
-   depmode=msvisualcpp
-fi
-
-if test "$depmode" = msvc7msys; then
-   # This is just like msvc7 but w/o cygpath translation.
-   # Just convert the backslash-escaped backslashes to single forward
-   # slashes to satisfy depend.m4
-   cygpath_u='sed s,\\\\,/,g'
-   depmode=msvc7
-fi
-
-if test "$depmode" = xlc; then
-   # IBM C/C++ Compilers xlc/xlC can output gcc-like dependency informations.
-   gccflag=-qmakedep=gcc,-MF
-   depmode=gcc
-fi
-
-case "$depmode" in
-gcc3)
-## gcc 3 implements dependency tracking that does exactly what
-## we want.  Yay!  Note: for some reason libtool 1.4 doesn't like
-## it if -MD -MP comes after the -MF stuff.  Hmm.
-## Unfortunately, FreeBSD c89 acceptance of flags depends upon
-## the command line argument order; so add the flags where they
-## appear in depend2.am.  Note that the slowdown incurred here
-## affects only configure: in makefiles, %FASTDEP% shortcuts this.
-  for arg
-  do
-    case $arg in
-    -c) set fnord "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" "$arg" ;;
-    *)  set fnord "$@" "$arg" ;;
-    esac
-    shift # fnord
-    shift # $arg
-  done
-  "$@"
-  stat=$?
-  if test $stat -eq 0; then :
-  else
-    rm -f "$tmpdepfile"
-    exit $stat
-  fi
-  mv "$tmpdepfile" "$depfile"
-  ;;
-
-gcc)
-## There are various ways to get dependency output from gcc.  Here's
-## why we pick this rather obscure method:
-## - Don't want to use -MD because we'd like the dependencies to end
-##   up in a subdir.  Having to rename by hand is ugly.
-##   (We might end up doing this anyway to support other compilers.)
-## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like
-##   -MM, not -M (despite what the docs say).
-## - Using -M directly means running the compiler twice (even worse
-##   than renaming).
-  if test -z "$gccflag"; then
-    gccflag=-MD,
-  fi
-  "$@" -Wp,"$gccflag$tmpdepfile"
-  stat=$?
-  if test $stat -eq 0; then :
-  else
-    rm -f "$tmpdepfile"
-    exit $stat
-  fi
-  rm -f "$depfile"
-  echo "$object : \\" > "$depfile"
-  alpha=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
-## The second -e expression handles DOS-style file names with drive letters.
-  sed -e 's/^[^:]*: / /' \
-      -e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile"
-## This next piece of magic avoids the "deleted header file" problem.
-## The problem is that when a header file which appears in a .P file
-## is deleted, the dependency causes make to die (because there is
-## typically no way to rebuild the header).  We avoid this by adding
-## dummy dependencies for each header file.  Too bad gcc doesn't do
-## this for us directly.
-  tr ' ' "$nl" < "$tmpdepfile" |
-## Some versions of gcc put a space before the ':'.  On the theory
-## that the space means something, we add a space to the output as
-## well.  hp depmode also adds that space, but also prefixes the VPATH
-## to the object.  Take care to not repeat it in the output.
-## Some versions of the HPUX 10.20 sed can't process this invocation
-## correctly.  Breaking it into two sed invocations is a workaround.
-    sed -e 's/^\\$//' -e '/^$/d' -e "s|.*$object$||" -e '/:$/d' \
-      | sed -e 's/$/ :/' >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-hp)
-  # This case exists only to let depend.m4 do its work.  It works by
-  # looking at the text of this script.  This case will never be run,
-  # since it is checked for above.
-  exit 1
-  ;;
-
-sgi)
-  if test "$libtool" = yes; then
-    "$@" "-Wp,-MDupdate,$tmpdepfile"
-  else
-    "$@" -MDupdate "$tmpdepfile"
-  fi
-  stat=$?
-  if test $stat -eq 0; then :
-  else
-    rm -f "$tmpdepfile"
-    exit $stat
-  fi
-  rm -f "$depfile"
-
-  if test -f "$tmpdepfile"; then  # yes, the sourcefile depend on other files
-    echo "$object : \\" > "$depfile"
-
-    # Clip off the initial element (the dependent).  Don't try to be
-    # clever and replace this with sed code, as IRIX sed won't handle
-    # lines with more than a fixed number of characters (4096 in
-    # IRIX 6.2 sed, 8192 in IRIX 6.5).  We also remove comment lines;
-    # the IRIX cc adds comments like '#:fec' to the end of the
-    # dependency line.
-    tr ' ' "$nl" < "$tmpdepfile" \
-    | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' | \
-    tr "$nl" ' ' >> "$depfile"
-    echo >> "$depfile"
-
-    # The second pass generates a dummy entry for each header file.
-    tr ' ' "$nl" < "$tmpdepfile" \
-   | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \
-   >> "$depfile"
-  else
-    # The sourcefile does not contain any dependencies, so just
-    # store a dummy comment line, to avoid errors with the Makefile
-    # "include basename.Plo" scheme.
-    echo "#dummy" > "$depfile"
-  fi
-  rm -f "$tmpdepfile"
-  ;;
-
-xlc)
-  # This case exists only to let depend.m4 do its work.  It works by
-  # looking at the text of this script.  This case will never be run,
-  # since it is checked for above.
-  exit 1
-  ;;
-
-aix)
-  # The C for AIX Compiler uses -M and outputs the dependencies
-  # in a .u file.  In older versions, this file always lives in the
-  # current directory.  Also, the AIX compiler puts '$object:' at the
-  # start of each line; $object doesn't have directory information.
-  # Version 6 uses the directory in both cases.
-  dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
-  test "x$dir" = "x$object" && dir=
-  base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
-  if test "$libtool" = yes; then
-    tmpdepfile1=$dir$base.u
-    tmpdepfile2=$base.u
-    tmpdepfile3=$dir.libs/$base.u
-    "$@" -Wc,-M
-  else
-    tmpdepfile1=$dir$base.u
-    tmpdepfile2=$dir$base.u
-    tmpdepfile3=$dir$base.u
-    "$@" -M
-  fi
-  stat=$?
-
-  if test $stat -eq 0; then :
-  else
-    rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
-    exit $stat
-  fi
-
-  for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
-  do
-    test -f "$tmpdepfile" && break
-  done
-  if test -f "$tmpdepfile"; then
-    # Each line is of the form 'foo.o: dependent.h'.
-    # Do two passes, one to just change these to
-    # '$object: dependent.h' and one to simply 'dependent.h:'.
-    sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
-    sed -e 's,^.*\.[a-z]*:['"$tab"' ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
-  else
-    # The sourcefile does not contain any dependencies, so just
-    # store a dummy comment line, to avoid errors with the Makefile
-    # "include basename.Plo" scheme.
-    echo "#dummy" > "$depfile"
-  fi
-  rm -f "$tmpdepfile"
-  ;;
-
-icc)
-  # Intel's C compiler anf tcc (Tiny C Compiler) understand '-MD -MF file'.
-  # However on
-  #    $CC -MD -MF foo.d -c -o sub/foo.o sub/foo.c
-  # ICC 7.0 will fill foo.d with something like
-  #    foo.o: sub/foo.c
-  #    foo.o: sub/foo.h
-  # which is wrong.  We want
-  #    sub/foo.o: sub/foo.c
-  #    sub/foo.o: sub/foo.h
-  #    sub/foo.c:
-  #    sub/foo.h:
-  # ICC 7.1 will output
-  #    foo.o: sub/foo.c sub/foo.h
-  # and will wrap long lines using '\':
-  #    foo.o: sub/foo.c ... \
-  #     sub/foo.h ... \
-  #     ...
-  # tcc 0.9.26 (FIXME still under development at the moment of writing)
-  # will emit a similar output, but also prepend the continuation lines
-  # with horizontal tabulation characters.
-  "$@" -MD -MF "$tmpdepfile"
-  stat=$?
-  if test $stat -eq 0; then :
-  else
-    rm -f "$tmpdepfile"
-    exit $stat
-  fi
-  rm -f "$depfile"
-  # Each line is of the form 'foo.o: dependent.h',
-  # or 'foo.o: dep1.h dep2.h \', or ' dep3.h dep4.h \'.
-  # Do two passes, one to just change these to
-  # '$object: dependent.h' and one to simply 'dependent.h:'.
-  sed -e "s/^[ $tab][ $tab]*/  /" -e "s,^[^:]*:,$object :," \
-    < "$tmpdepfile" > "$depfile"
-  sed '
-    s/[ '"$tab"'][ '"$tab"']*/ /g
-    s/^ *//
-    s/ *\\*$//
-    s/^[^:]*: *//
-    /^$/d
-    /:$/d
-    s/$/ :/
-  ' < "$tmpdepfile" >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-hp2)
-  # The "hp" stanza above does not work with aCC (C++) and HP's ia64
-  # compilers, which have integrated preprocessors.  The correct option
-  # to use with these is +Maked; it writes dependencies to a file named
-  # 'foo.d', which lands next to the object file, wherever that
-  # happens to be.
-  # Much of this is similar to the tru64 case; see comments there.
-  dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
-  test "x$dir" = "x$object" && dir=
-  base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
-  if test "$libtool" = yes; then
-    tmpdepfile1=$dir$base.d
-    tmpdepfile2=$dir.libs/$base.d
-    "$@" -Wc,+Maked
-  else
-    tmpdepfile1=$dir$base.d
-    tmpdepfile2=$dir$base.d
-    "$@" +Maked
-  fi
-  stat=$?
-  if test $stat -eq 0; then :
-  else
-     rm -f "$tmpdepfile1" "$tmpdepfile2"
-     exit $stat
-  fi
-
-  for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2"
-  do
-    test -f "$tmpdepfile" && break
-  done
-  if test -f "$tmpdepfile"; then
-    sed -e "s,^.*\.[a-z]*:,$object:," "$tmpdepfile" > "$depfile"
-    # Add 'dependent.h:' lines.
-    sed -ne '2,${
-	       s/^ *//
-	       s/ \\*$//
-	       s/$/:/
-	       p
-	     }' "$tmpdepfile" >> "$depfile"
-  else
-    echo "#dummy" > "$depfile"
-  fi
-  rm -f "$tmpdepfile" "$tmpdepfile2"
-  ;;
-
-tru64)
-   # The Tru64 compiler uses -MD to generate dependencies as a side
-   # effect.  'cc -MD -o foo.o ...' puts the dependencies into 'foo.o.d'.
-   # At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put
-   # dependencies in 'foo.d' instead, so we check for that too.
-   # Subdirectories are respected.
-   dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
-   test "x$dir" = "x$object" && dir=
-   base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
-
-   if test "$libtool" = yes; then
-      # With Tru64 cc, shared objects can also be used to make a
-      # static library.  This mechanism is used in libtool 1.4 series to
-      # handle both shared and static libraries in a single compilation.
-      # With libtool 1.4, dependencies were output in $dir.libs/$base.lo.d.
-      #
-      # With libtool 1.5 this exception was removed, and libtool now
-      # generates 2 separate objects for the 2 libraries.  These two
-      # compilations output dependencies in $dir.libs/$base.o.d and
-      # in $dir$base.o.d.  We have to check for both files, because
-      # one of the two compilations can be disabled.  We should prefer
-      # $dir$base.o.d over $dir.libs/$base.o.d because the latter is
-      # automatically cleaned when .libs/ is deleted, while ignoring
-      # the former would cause a distcleancheck panic.
-      tmpdepfile1=$dir.libs/$base.lo.d   # libtool 1.4
-      tmpdepfile2=$dir$base.o.d          # libtool 1.5
-      tmpdepfile3=$dir.libs/$base.o.d    # libtool 1.5
-      tmpdepfile4=$dir.libs/$base.d      # Compaq CCC V6.2-504
-      "$@" -Wc,-MD
-   else
-      tmpdepfile1=$dir$base.o.d
-      tmpdepfile2=$dir$base.d
-      tmpdepfile3=$dir$base.d
-      tmpdepfile4=$dir$base.d
-      "$@" -MD
-   fi
-
-   stat=$?
-   if test $stat -eq 0; then :
-   else
-      rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
-      exit $stat
-   fi
-
-   for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
-   do
-     test -f "$tmpdepfile" && break
-   done
-   if test -f "$tmpdepfile"; then
-      sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
-      sed -e 's,^.*\.[a-z]*:['"$tab"' ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
-   else
-      echo "#dummy" > "$depfile"
-   fi
-   rm -f "$tmpdepfile"
-   ;;
-
-msvc7)
-  if test "$libtool" = yes; then
-    showIncludes=-Wc,-showIncludes
-  else
-    showIncludes=-showIncludes
-  fi
-  "$@" $showIncludes > "$tmpdepfile"
-  stat=$?
-  grep -v '^Note: including file: ' "$tmpdepfile"
-  if test "$stat" = 0; then :
-  else
-    rm -f "$tmpdepfile"
-    exit $stat
-  fi
-  rm -f "$depfile"
-  echo "$object : \\" > "$depfile"
-  # The first sed program below extracts the file names and escapes
-  # backslashes for cygpath.  The second sed program outputs the file
-  # name when reading, but also accumulates all include files in the
-  # hold buffer in order to output them again at the end.  This only
-  # works with sed implementations that can handle large buffers.
-  sed < "$tmpdepfile" -n '
-/^Note: including file:  *\(.*\)/ {
-  s//\1/
-  s/\\/\\\\/g
-  p
-}' | $cygpath_u | sort -u | sed -n '
-s/ /\\ /g
-s/\(.*\)/'"$tab"'\1 \\/p
-s/.\(.*\) \\/\1:/
-H
-$ {
-  s/.*/'"$tab"'/
-  G
-  p
-}' >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-msvc7msys)
-  # This case exists only to let depend.m4 do its work.  It works by
-  # looking at the text of this script.  This case will never be run,
-  # since it is checked for above.
-  exit 1
-  ;;
-
-#nosideeffect)
-  # This comment above is used by automake to tell side-effect
-  # dependency tracking mechanisms from slower ones.
-
-dashmstdout)
-  # Important note: in order to support this mode, a compiler *must*
-  # always write the preprocessed file to stdout, regardless of -o.
-  "$@" || exit $?
-
-  # Remove the call to Libtool.
-  if test "$libtool" = yes; then
-    while test "X$1" != 'X--mode=compile'; do
-      shift
-    done
-    shift
-  fi
-
-  # Remove '-o $object'.
-  IFS=" "
-  for arg
-  do
-    case $arg in
-    -o)
-      shift
-      ;;
-    $object)
-      shift
-      ;;
-    *)
-      set fnord "$@" "$arg"
-      shift # fnord
-      shift # $arg
-      ;;
-    esac
-  done
-
-  test -z "$dashmflag" && dashmflag=-M
-  # Require at least two characters before searching for ':'
-  # in the target name.  This is to cope with DOS-style filenames:
-  # a dependency such as 'c:/foo/bar' could be seen as target 'c' otherwise.
-  "$@" $dashmflag |
-    sed 's:^['"$tab"' ]*[^:'"$tab"' ][^:][^:]*\:['"$tab"' ]*:'"$object"'\: :' > "$tmpdepfile"
-  rm -f "$depfile"
-  cat < "$tmpdepfile" > "$depfile"
-  tr ' ' "$nl" < "$tmpdepfile" | \
-## Some versions of the HPUX 10.20 sed can't process this invocation
-## correctly.  Breaking it into two sed invocations is a workaround.
-    sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-dashXmstdout)
-  # This case only exists to satisfy depend.m4.  It is never actually
-  # run, as this mode is specially recognized in the preamble.
-  exit 1
-  ;;
-
-makedepend)
-  "$@" || exit $?
-  # Remove any Libtool call
-  if test "$libtool" = yes; then
-    while test "X$1" != 'X--mode=compile'; do
-      shift
-    done
-    shift
-  fi
-  # X makedepend
-  shift
-  cleared=no eat=no
-  for arg
-  do
-    case $cleared in
-    no)
-      set ""; shift
-      cleared=yes ;;
-    esac
-    if test $eat = yes; then
-      eat=no
-      continue
-    fi
-    case "$arg" in
-    -D*|-I*)
-      set fnord "$@" "$arg"; shift ;;
-    # Strip any option that makedepend may not understand.  Remove
-    # the object too, otherwise makedepend will parse it as a source file.
-    -arch)
-      eat=yes ;;
-    -*|$object)
-      ;;
-    *)
-      set fnord "$@" "$arg"; shift ;;
-    esac
-  done
-  obj_suffix=`echo "$object" | sed 's/^.*\././'`
-  touch "$tmpdepfile"
-  ${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@"
-  rm -f "$depfile"
-  # makedepend may prepend the VPATH from the source file name to the object.
-  # No need to regex-escape $object, excess matching of '.' is harmless.
-  sed "s|^.*\($object *:\)|\1|" "$tmpdepfile" > "$depfile"
-  sed '1,2d' "$tmpdepfile" | tr ' ' "$nl" | \
-## Some versions of the HPUX 10.20 sed can't process this invocation
-## correctly.  Breaking it into two sed invocations is a workaround.
-    sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
-  rm -f "$tmpdepfile" "$tmpdepfile".bak
-  ;;
-
-cpp)
-  # Important note: in order to support this mode, a compiler *must*
-  # always write the preprocessed file to stdout.
-  "$@" || exit $?
-
-  # Remove the call to Libtool.
-  if test "$libtool" = yes; then
-    while test "X$1" != 'X--mode=compile'; do
-      shift
-    done
-    shift
-  fi
-
-  # Remove '-o $object'.
-  IFS=" "
-  for arg
-  do
-    case $arg in
-    -o)
-      shift
-      ;;
-    $object)
-      shift
-      ;;
-    *)
-      set fnord "$@" "$arg"
-      shift # fnord
-      shift # $arg
-      ;;
-    esac
-  done
-
-  "$@" -E |
-    sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \
-       -e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' |
-    sed '$ s: \\$::' > "$tmpdepfile"
-  rm -f "$depfile"
-  echo "$object : \\" > "$depfile"
-  cat < "$tmpdepfile" >> "$depfile"
-  sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-msvisualcpp)
-  # Important note: in order to support this mode, a compiler *must*
-  # always write the preprocessed file to stdout.
-  "$@" || exit $?
-
-  # Remove the call to Libtool.
-  if test "$libtool" = yes; then
-    while test "X$1" != 'X--mode=compile'; do
-      shift
-    done
-    shift
-  fi
-
-  IFS=" "
-  for arg
-  do
-    case "$arg" in
-    -o)
-      shift
-      ;;
-    $object)
-      shift
-      ;;
-    "-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI")
-	set fnord "$@"
-	shift
-	shift
-	;;
-    *)
-	set fnord "$@" "$arg"
-	shift
-	shift
-	;;
-    esac
-  done
-  "$@" -E 2>/dev/null |
-  sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::\1:p' | $cygpath_u | sort -u > "$tmpdepfile"
-  rm -f "$depfile"
-  echo "$object : \\" > "$depfile"
-  sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::'"$tab"'\1 \\:p' >> "$depfile"
-  echo "$tab" >> "$depfile"
-  sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::\1\::p' >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-msvcmsys)
-  # This case exists only to let depend.m4 do its work.  It works by
-  # looking at the text of this script.  This case will never be run,
-  # since it is checked for above.
-  exit 1
-  ;;
-
-none)
-  exec "$@"
-  ;;
-
-*)
-  echo "Unknown depmode $depmode" 1>&2
-  exit 1
-  ;;
-esac
-
-exit 0
-
-# Local Variables:
-# mode: shell-script
-# sh-indentation: 2
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
-# time-stamp-end: "; # UTC"
-# End:
@@ -1,527 +0,0 @@
-#!/bin/sh
-# install - install a program, script, or datafile
-
-scriptversion=2011-01-19.21; # UTC
-
-# This originates from X11R5 (mit/util/scripts/install.sh), which was
-# later released in X11R6 (xc/config/util/install.sh) with the
-# following copyright and license.
-#
-# Copyright (C) 1994 X Consortium
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to
-# deal in the Software without restriction, including without limitation the
-# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-# sell copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
-# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC-
-# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#
-# Except as contained in this notice, the name of the X Consortium shall not
-# be used in advertising or otherwise to promote the sale, use or other deal-
-# ings in this Software without prior written authorization from the X Consor-
-# tium.
-#
-#
-# FSF changes to this file are in the public domain.
-#
-# Calling this script install-sh is preferred over install.sh, to prevent
-# `make' implicit rules from creating a file called install from it
-# when there is no Makefile.
-#
-# This script is compatible with the BSD install script, but was written
-# from scratch.
-
-nl='
-'
-IFS=" ""	$nl"
-
-# set DOITPROG to echo to test this script
-
-# Don't use :- since 4.3BSD and earlier shells don't like it.
-doit=${DOITPROG-}
-if test -z "$doit"; then
-  doit_exec=exec
-else
-  doit_exec=$doit
-fi
-
-# Put in absolute file names if you don't have them in your path;
-# or use environment vars.
-
-chgrpprog=${CHGRPPROG-chgrp}
-chmodprog=${CHMODPROG-chmod}
-chownprog=${CHOWNPROG-chown}
-cmpprog=${CMPPROG-cmp}
-cpprog=${CPPROG-cp}
-mkdirprog=${MKDIRPROG-mkdir}
-mvprog=${MVPROG-mv}
-rmprog=${RMPROG-rm}
-stripprog=${STRIPPROG-strip}
-
-posix_glob='?'
-initialize_posix_glob='
-  test "$posix_glob" != "?" || {
-    if (set -f) 2>/dev/null; then
-      posix_glob=
-    else
-      posix_glob=:
-    fi
-  }
-'
-
-posix_mkdir=
-
-# Desired mode of installed file.
-mode=0755
-
-chgrpcmd=
-chmodcmd=$chmodprog
-chowncmd=
-mvcmd=$mvprog
-rmcmd="$rmprog -f"
-stripcmd=
-
-src=
-dst=
-dir_arg=
-dst_arg=
-
-copy_on_change=false
-no_target_directory=
-
-usage="\
-Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
-   or: $0 [OPTION]... SRCFILES... DIRECTORY
-   or: $0 [OPTION]... -t DIRECTORY SRCFILES...
-   or: $0 [OPTION]... -d DIRECTORIES...
-
-In the 1st form, copy SRCFILE to DSTFILE.
-In the 2nd and 3rd, copy all SRCFILES to DIRECTORY.
-In the 4th, create DIRECTORIES.
-
-Options:
-     --help     display this help and exit.
-     --version  display version info and exit.
-
-  -c            (ignored)
-  -C            install only if different (preserve the last data modification time)
-  -d            create directories instead of installing files.
-  -g GROUP      $chgrpprog installed files to GROUP.
-  -m MODE       $chmodprog installed files to MODE.
-  -o USER       $chownprog installed files to USER.
-  -s            $stripprog installed files.
-  -t DIRECTORY  install into DIRECTORY.
-  -T            report an error if DSTFILE is a directory.
-
-Environment variables override the default commands:
-  CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG
-  RMPROG STRIPPROG
-"
-
-while test $# -ne 0; do
-  case $1 in
-    -c) ;;
-
-    -C) copy_on_change=true;;
-
-    -d) dir_arg=true;;
-
-    -g) chgrpcmd="$chgrpprog $2"
-	shift;;
-
-    --help) echo "$usage"; exit $?;;
-
-    -m) mode=$2
-	case $mode in
-	  *' '* | *'	'* | *'
-'*	  | *'*'* | *'?'* | *'['*)
-	    echo "$0: invalid mode: $mode" >&2
-	    exit 1;;
-	esac
-	shift;;
-
-    -o) chowncmd="$chownprog $2"
-	shift;;
-
-    -s) stripcmd=$stripprog;;
-
-    -t) dst_arg=$2
-	# Protect names problematic for `test' and other utilities.
-	case $dst_arg in
-	  -* | [=\(\)!]) dst_arg=./$dst_arg;;
-	esac
-	shift;;
-
-    -T) no_target_directory=true;;
-
-    --version) echo "$0 $scriptversion"; exit $?;;
-
-    --)	shift
-	break;;
-
-    -*)	echo "$0: invalid option: $1" >&2
-	exit 1;;
-
-    *)  break;;
-  esac
-  shift
-done
-
-if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
-  # When -d is used, all remaining arguments are directories to create.
-  # When -t is used, the destination is already specified.
-  # Otherwise, the last argument is the destination.  Remove it from $@.
-  for arg
-  do
-    if test -n "$dst_arg"; then
-      # $@ is not empty: it contains at least $arg.
-      set fnord "$@" "$dst_arg"
-      shift # fnord
-    fi
-    shift # arg
-    dst_arg=$arg
-    # Protect names problematic for `test' and other utilities.
-    case $dst_arg in
-      -* | [=\(\)!]) dst_arg=./$dst_arg;;
-    esac
-  done
-fi
-
-if test $# -eq 0; then
-  if test -z "$dir_arg"; then
-    echo "$0: no input file specified." >&2
-    exit 1
-  fi
-  # It's OK to call `install-sh -d' without argument.
-  # This can happen when creating conditional directories.
-  exit 0
-fi
-
-if test -z "$dir_arg"; then
-  do_exit='(exit $ret); exit $ret'
-  trap "ret=129; $do_exit" 1
-  trap "ret=130; $do_exit" 2
-  trap "ret=141; $do_exit" 13
-  trap "ret=143; $do_exit" 15
-
-  # Set umask so as not to create temps with too-generous modes.
-  # However, 'strip' requires both read and write access to temps.
-  case $mode in
-    # Optimize common cases.
-    *644) cp_umask=133;;
-    *755) cp_umask=22;;
-
-    *[0-7])
-      if test -z "$stripcmd"; then
-	u_plus_rw=
-      else
-	u_plus_rw='% 200'
-      fi
-      cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
-    *)
-      if test -z "$stripcmd"; then
-	u_plus_rw=
-      else
-	u_plus_rw=,u+rw
-      fi
-      cp_umask=$mode$u_plus_rw;;
-  esac
-fi
-
-for src
-do
-  # Protect names problematic for `test' and other utilities.
-  case $src in
-    -* | [=\(\)!]) src=./$src;;
-  esac
-
-  if test -n "$dir_arg"; then
-    dst=$src
-    dstdir=$dst
-    test -d "$dstdir"
-    dstdir_status=$?
-  else
-
-    # Waiting for this to be detected by the "$cpprog $src $dsttmp" command
-    # might cause directories to be created, which would be especially bad
-    # if $src (and thus $dsttmp) contains '*'.
-    if test ! -f "$src" && test ! -d "$src"; then
-      echo "$0: $src does not exist." >&2
-      exit 1
-    fi
-
-    if test -z "$dst_arg"; then
-      echo "$0: no destination specified." >&2
-      exit 1
-    fi
-    dst=$dst_arg
-
-    # If destination is a directory, append the input filename; won't work
-    # if double slashes aren't ignored.
-    if test -d "$dst"; then
-      if test -n "$no_target_directory"; then
-	echo "$0: $dst_arg: Is a directory" >&2
-	exit 1
-      fi
-      dstdir=$dst
-      dst=$dstdir/`basename "$src"`
-      dstdir_status=0
-    else
-      # Prefer dirname, but fall back on a substitute if dirname fails.
-      dstdir=`
-	(dirname "$dst") 2>/dev/null ||
-	expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
-	     X"$dst" : 'X\(//\)[^/]' \| \
-	     X"$dst" : 'X\(//\)$' \| \
-	     X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
-	echo X"$dst" |
-	    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\/\)[^/].*/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\/\)$/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\).*/{
-		   s//\1/
-		   q
-		 }
-		 s/.*/./; q'
-      `
-
-      test -d "$dstdir"
-      dstdir_status=$?
-    fi
-  fi
-
-  obsolete_mkdir_used=false
-
-  if test $dstdir_status != 0; then
-    case $posix_mkdir in
-      '')
-	# Create intermediate dirs using mode 755 as modified by the umask.
-	# This is like FreeBSD 'install' as of 1997-10-28.
-	umask=`umask`
-	case $stripcmd.$umask in
-	  # Optimize common cases.
-	  *[2367][2367]) mkdir_umask=$umask;;
-	  .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
-
-	  *[0-7])
-	    mkdir_umask=`expr $umask + 22 \
-	      - $umask % 100 % 40 + $umask % 20 \
-	      - $umask % 10 % 4 + $umask % 2
-	    `;;
-	  *) mkdir_umask=$umask,go-w;;
-	esac
-
-	# With -d, create the new directory with the user-specified mode.
-	# Otherwise, rely on $mkdir_umask.
-	if test -n "$dir_arg"; then
-	  mkdir_mode=-m$mode
-	else
-	  mkdir_mode=
-	fi
-
-	posix_mkdir=false
-	case $umask in
-	  *[123567][0-7][0-7])
-	    # POSIX mkdir -p sets u+wx bits regardless of umask, which
-	    # is incompatible with FreeBSD 'install' when (umask & 300) != 0.
-	    ;;
-	  *)
-	    tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
-	    trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
-
-	    if (umask $mkdir_umask &&
-		exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
-	    then
-	      if test -z "$dir_arg" || {
-		   # Check for POSIX incompatibilities with -m.
-		   # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
-		   # other-writeable bit of parent directory when it shouldn't.
-		   # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
-		   ls_ld_tmpdir=`ls -ld "$tmpdir"`
-		   case $ls_ld_tmpdir in
-		     d????-?r-*) different_mode=700;;
-		     d????-?--*) different_mode=755;;
-		     *) false;;
-		   esac &&
-		   $mkdirprog -m$different_mode -p -- "$tmpdir" && {
-		     ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
-		     test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
-		   }
-		 }
-	      then posix_mkdir=:
-	      fi
-	      rmdir "$tmpdir/d" "$tmpdir"
-	    else
-	      # Remove any dirs left behind by ancient mkdir implementations.
-	      rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
-	    fi
-	    trap '' 0;;
-	esac;;
-    esac
-
-    if
-      $posix_mkdir && (
-	umask $mkdir_umask &&
-	$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
-      )
-    then :
-    else
-
-      # The umask is ridiculous, or mkdir does not conform to POSIX,
-      # or it failed possibly due to a race condition.  Create the
-      # directory the slow way, step by step, checking for races as we go.
-
-      case $dstdir in
-	/*) prefix='/';;
-	[-=\(\)!]*) prefix='./';;
-	*)  prefix='';;
-      esac
-
-      eval "$initialize_posix_glob"
-
-      oIFS=$IFS
-      IFS=/
-      $posix_glob set -f
-      set fnord $dstdir
-      shift
-      $posix_glob set +f
-      IFS=$oIFS
-
-      prefixes=
-
-      for d
-      do
-	test X"$d" = X && continue
-
-	prefix=$prefix$d
-	if test -d "$prefix"; then
-	  prefixes=
-	else
-	  if $posix_mkdir; then
-	    (umask=$mkdir_umask &&
-	     $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
-	    # Don't fail if two instances are running concurrently.
-	    test -d "$prefix" || exit 1
-	  else
-	    case $prefix in
-	      *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
-	      *) qprefix=$prefix;;
-	    esac
-	    prefixes="$prefixes '$qprefix'"
-	  fi
-	fi
-	prefix=$prefix/
-      done
-
-      if test -n "$prefixes"; then
-	# Don't fail if two instances are running concurrently.
-	(umask $mkdir_umask &&
-	 eval "\$doit_exec \$mkdirprog $prefixes") ||
-	  test -d "$dstdir" || exit 1
-	obsolete_mkdir_used=true
-      fi
-    fi
-  fi
-
-  if test -n "$dir_arg"; then
-    { test -z "$chowncmd" || $doit $chowncmd "$dst"; } &&
-    { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } &&
-    { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false ||
-      test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1
-  else
-
-    # Make a couple of temp file names in the proper directory.
-    dsttmp=$dstdir/_inst.$$_
-    rmtmp=$dstdir/_rm.$$_
-
-    # Trap to clean up those temp files at exit.
-    trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0
-
-    # Copy the file name to the temp name.
-    (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") &&
-
-    # and set any options; do chmod last to preserve setuid bits.
-    #
-    # If any of these fail, we abort the whole thing.  If we want to
-    # ignore errors from any of these, just make sure not to ignore
-    # errors from the above "$doit $cpprog $src $dsttmp" command.
-    #
-    { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } &&
-    { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } &&
-    { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } &&
-    { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
-
-    # If -C, don't bother to copy if it wouldn't change the file.
-    if $copy_on_change &&
-       old=`LC_ALL=C ls -dlL "$dst"	2>/dev/null` &&
-       new=`LC_ALL=C ls -dlL "$dsttmp"	2>/dev/null` &&
-
-       eval "$initialize_posix_glob" &&
-       $posix_glob set -f &&
-       set X $old && old=:$2:$4:$5:$6 &&
-       set X $new && new=:$2:$4:$5:$6 &&
-       $posix_glob set +f &&
-
-       test "$old" = "$new" &&
-       $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
-    then
-      rm -f "$dsttmp"
-    else
-      # Rename the file to the real destination.
-      $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null ||
-
-      # The rename failed, perhaps because mv can't rename something else
-      # to itself, or perhaps because mv is so ancient that it does not
-      # support -f.
-      {
-	# Now remove or move aside any old file at destination location.
-	# We try this two ways since rm can't unlink itself on some
-	# systems and the destination file might be busy for other
-	# reasons.  In this case, the final cleanup might fail but the new
-	# file should still install successfully.
-	{
-	  test ! -f "$dst" ||
-	  $doit $rmcmd -f "$dst" 2>/dev/null ||
-	  { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
-	    { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }
-	  } ||
-	  { echo "$0: cannot unlink or rename $dst" >&2
-	    (exit 1); exit 1
-	  }
-	} &&
-
-	# Now rename the file to the real destination.
-	$doit $mvcmd "$dsttmp" "$dst"
-      }
-    fi || exit 1
-
-    trap '' 0
-  fi
-done
-
-# Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
-# time-stamp-end: "; # UTC"
-# End:
@@ -1,331 +0,0 @@
-#! /bin/sh
-# Common stub for a few missing GNU programs while installing.
-
-scriptversion=2012-01-06.13; # UTC
-
-# Copyright (C) 1996, 1997, 1999, 2000, 2002, 2003, 2004, 2005, 2006,
-# 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
-# Originally by Fran,cois Pinard <pinard@iro.umontreal.ca>, 1996.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# As a special exception to the GNU General Public License, if you
-# distribute this file as part of a program that contains a
-# configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
-
-if test $# -eq 0; then
-  echo 1>&2 "Try \`$0 --help' for more information"
-  exit 1
-fi
-
-run=:
-sed_output='s/.* --output[ =]\([^ ]*\).*/\1/p'
-sed_minuso='s/.* -o \([^ ]*\).*/\1/p'
-
-# In the cases where this matters, `missing' is being run in the
-# srcdir already.
-if test -f configure.ac; then
-  configure_ac=configure.ac
-else
-  configure_ac=configure.in
-fi
-
-msg="missing on your system"
-
-case $1 in
--run)
-  # Try to run requested program, and just exit if it succeeds.
-  run=
-  shift
-  "$@" && exit 0
-  # Exit code 63 means version mismatch.  This often happens
-  # when the user try to use an ancient version of a tool on
-  # a file that requires a minimum version.  In this case we
-  # we should proceed has if the program had been absent, or
-  # if --run hadn't been passed.
-  if test $? = 63; then
-    run=:
-    msg="probably too old"
-  fi
-  ;;
-
-  -h|--h|--he|--hel|--help)
-    echo "\
-$0 [OPTION]... PROGRAM [ARGUMENT]...
-
-Handle \`PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an
-error status if there is no known handling for PROGRAM.
-
-Options:
-  -h, --help      display this help and exit
-  -v, --version   output version information and exit
-  --run           try to run the given command, and emulate it if it fails
-
-Supported PROGRAM values:
-  aclocal      touch file \`aclocal.m4'
-  autoconf     touch file \`configure'
-  autoheader   touch file \`config.h.in'
-  autom4te     touch the output file, or create a stub one
-  automake     touch all \`Makefile.in' files
-  bison        create \`y.tab.[ch]', if possible, from existing .[ch]
-  flex         create \`lex.yy.c', if possible, from existing .c
-  help2man     touch the output file
-  lex          create \`lex.yy.c', if possible, from existing .c
-  makeinfo     touch the output file
-  yacc         create \`y.tab.[ch]', if possible, from existing .[ch]
-
-Version suffixes to PROGRAM as well as the prefixes \`gnu-', \`gnu', and
-\`g' are ignored when checking the name.
-
-Send bug reports to <bug-automake@gnu.org>."
-    exit $?
-    ;;
-
-  -v|--v|--ve|--ver|--vers|--versi|--versio|--version)
-    echo "missing $scriptversion (GNU Automake)"
-    exit $?
-    ;;
-
-  -*)
-    echo 1>&2 "$0: Unknown \`$1' option"
-    echo 1>&2 "Try \`$0 --help' for more information"
-    exit 1
-    ;;
-
-esac
-
-# normalize program name to check for.
-program=`echo "$1" | sed '
-  s/^gnu-//; t
-  s/^gnu//; t
-  s/^g//; t'`
-
-# Now exit if we have it, but it failed.  Also exit now if we
-# don't have it and --version was passed (most likely to detect
-# the program).  This is about non-GNU programs, so use $1 not
-# $program.
-case $1 in
-  lex*|yacc*)
-    # Not GNU programs, they don't have --version.
-    ;;
-
-  *)
-    if test -z "$run" && ($1 --version) > /dev/null 2>&1; then
-       # We have it, but it failed.
-       exit 1
-    elif test "x$2" = "x--version" || test "x$2" = "x--help"; then
-       # Could not run --version or --help.  This is probably someone
-       # running `$TOOL --version' or `$TOOL --help' to check whether
-       # $TOOL exists and not knowing $TOOL uses missing.
-       exit 1
-    fi
-    ;;
-esac
-
-# If it does not exist, or fails to run (possibly an outdated version),
-# try to emulate it.
-case $program in
-  aclocal*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified \`acinclude.m4' or \`${configure_ac}'.  You might want
-         to install the \`Automake' and \`Perl' packages.  Grab them from
-         any GNU archive site."
-    touch aclocal.m4
-    ;;
-
-  autoconf*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified \`${configure_ac}'.  You might want to install the
-         \`Autoconf' and \`GNU m4' packages.  Grab them from any GNU
-         archive site."
-    touch configure
-    ;;
-
-  autoheader*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified \`acconfig.h' or \`${configure_ac}'.  You might want
-         to install the \`Autoconf' and \`GNU m4' packages.  Grab them
-         from any GNU archive site."
-    files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER(\([^)]*\)).*/\1/p' ${configure_ac}`
-    test -z "$files" && files="config.h"
-    touch_files=
-    for f in $files; do
-      case $f in
-      *:*) touch_files="$touch_files "`echo "$f" |
-				       sed -e 's/^[^:]*://' -e 's/:.*//'`;;
-      *) touch_files="$touch_files $f.in";;
-      esac
-    done
-    touch $touch_files
-    ;;
-
-  automake*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified \`Makefile.am', \`acinclude.m4' or \`${configure_ac}'.
-         You might want to install the \`Automake' and \`Perl' packages.
-         Grab them from any GNU archive site."
-    find . -type f -name Makefile.am -print |
-	   sed 's/\.am$/.in/' |
-	   while read f; do touch "$f"; done
-    ;;
-
-  autom4te*)
-    echo 1>&2 "\
-WARNING: \`$1' is needed, but is $msg.
-         You might have modified some files without having the
-         proper tools for further handling them.
-         You can get \`$1' as part of \`Autoconf' from any GNU
-         archive site."
-
-    file=`echo "$*" | sed -n "$sed_output"`
-    test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
-    if test -f "$file"; then
-	touch $file
-    else
-	test -z "$file" || exec >$file
-	echo "#! /bin/sh"
-	echo "# Created by GNU Automake missing as a replacement of"
-	echo "#  $ $@"
-	echo "exit 0"
-	chmod +x $file
-	exit 1
-    fi
-    ;;
-
-  bison*|yacc*)
-    echo 1>&2 "\
-WARNING: \`$1' $msg.  You should only need it if
-         you modified a \`.y' file.  You may need the \`Bison' package
-         in order for those modifications to take effect.  You can get
-         \`Bison' from any GNU archive site."
-    rm -f y.tab.c y.tab.h
-    if test $# -ne 1; then
-        eval LASTARG=\${$#}
-	case $LASTARG in
-	*.y)
-	    SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'`
-	    if test -f "$SRCFILE"; then
-	         cp "$SRCFILE" y.tab.c
-	    fi
-	    SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'`
-	    if test -f "$SRCFILE"; then
-	         cp "$SRCFILE" y.tab.h
-	    fi
-	  ;;
-	esac
-    fi
-    if test ! -f y.tab.h; then
-	echo >y.tab.h
-    fi
-    if test ! -f y.tab.c; then
-	echo 'main() { return 0; }' >y.tab.c
-    fi
-    ;;
-
-  lex*|flex*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified a \`.l' file.  You may need the \`Flex' package
-         in order for those modifications to take effect.  You can get
-         \`Flex' from any GNU archive site."
-    rm -f lex.yy.c
-    if test $# -ne 1; then
-        eval LASTARG=\${$#}
-	case $LASTARG in
-	*.l)
-	    SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'`
-	    if test -f "$SRCFILE"; then
-	         cp "$SRCFILE" lex.yy.c
-	    fi
-	  ;;
-	esac
-    fi
-    if test ! -f lex.yy.c; then
-	echo 'main() { return 0; }' >lex.yy.c
-    fi
-    ;;
-
-  help2man*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-	 you modified a dependency of a manual page.  You may need the
-	 \`Help2man' package in order for those modifications to take
-	 effect.  You can get \`Help2man' from any GNU archive site."
-
-    file=`echo "$*" | sed -n "$sed_output"`
-    test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
-    if test -f "$file"; then
-	touch $file
-    else
-	test -z "$file" || exec >$file
-	echo ".ab help2man is required to generate this page"
-	exit $?
-    fi
-    ;;
-
-  makeinfo*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified a \`.texi' or \`.texinfo' file, or any other file
-         indirectly affecting the aspect of the manual.  The spurious
-         call might also be the consequence of using a buggy \`make' (AIX,
-         DU, IRIX).  You might want to install the \`Texinfo' package or
-         the \`GNU make' package.  Grab either from any GNU archive site."
-    # The file to touch is that specified with -o ...
-    file=`echo "$*" | sed -n "$sed_output"`
-    test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
-    if test -z "$file"; then
-      # ... or it is the one specified with @setfilename ...
-      infile=`echo "$*" | sed 's/.* \([^ ]*\) *$/\1/'`
-      file=`sed -n '
-	/^@setfilename/{
-	  s/.* \([^ ]*\) *$/\1/
-	  p
-	  q
-	}' $infile`
-      # ... or it is derived from the source name (dir/f.texi becomes f.info)
-      test -z "$file" && file=`echo "$infile" | sed 's,.*/,,;s,.[^.]*$,,'`.info
-    fi
-    # If the file does not exist, the user really needs makeinfo;
-    # let's fail without touching anything.
-    test -f $file || exit 1
-    touch $file
-    ;;
-
-  *)
-    echo 1>&2 "\
-WARNING: \`$1' is needed, and is $msg.
-         You might have modified some files without having the
-         proper tools for further handling them.  Check the \`README' file,
-         it often tells you about the needed prerequisites for installing
-         this package.  You may also peek at any GNU archive site, in case
-         some other package would contain this missing \`$1' program."
-    exit 1
-    ;;
-esac
-
-exit 0
-
-# Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
-# time-stamp-end: "; # UTC"
-# End:
@@ -603,8 +603,8 @@ SubstituteFile(
 	sp = fopen(substitutions, "rt");
 	if (sp != NULL) {
 	    while (fgets(szBuffer, cbBuffer, sp) != NULL) {
-		char *ks, *ke, *vs, *ve;
-		ks = szBuffer;
+		unsigned char *ks, *ke, *vs, *ve;
+		ks = (unsigned char*)szBuffer;
 		while (ks && *ks && isspace(*ks)) ++ks;
 		ke = ks;
 		while (ke && *ke && !isspace(*ke)) ++ke;
@@ -613,7 +613,7 @@ SubstituteFile(
 		ve = vs;
 		while (ve && *ve && !(*ve == '\r' || *ve == '\n')) ++ve;
 		*ke = 0, *ve = 0;
-		list_insert(&substPtr, ks, vs);
+		list_insert(&substPtr, (char*)ks, (char*)vs);
 	    }
 	    fclose(sp);
 	}
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.9.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.11.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.9.0'
-PACKAGE_STRING='sqlite 3.9.0'
+PACKAGE_VERSION='3.11.0'
+PACKAGE_STRING='sqlite 3.11.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -774,6 +774,7 @@ USE_GCOV
 OPT_FEATURE_FLAGS
 USE_AMALGAMATION
 TARGET_DEBUG
+TARGET_HAVE_EDITLINE
 TARGET_HAVE_READLINE
 TARGET_READLINE_INC
 TARGET_READLINE_LIBS
@@ -803,7 +804,6 @@ VERSION
 program_prefix
 TCLLIBDIR
 TCLSH_CMD
-AWK
 INSTALL_DATA
 INSTALL_SCRIPT
 INSTALL_PROGRAM
@@ -896,6 +896,7 @@ enable_releasemode
 enable_tempstore
 enable_tcl
 with_tcl
+enable_editline
 enable_readline
 with_readline_lib
 with_readline_inc
@@ -1459,7 +1460,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.9.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.11.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1524,7 +1525,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.9.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.11.0:";;
   esac
  cat <<\_ACEOF

@@ -1543,7 +1544,8 @@ Optional Features:
  --enable-tempstore      Use an in-ram database for temporary tables
                          (never,no,yes,always)
  --disable-tcl           do not build TCL extension
-  --disable-readline      disable readline support [default=detect]
+  --enable-editline       enable BSD editline support
+  --disable-readline      disable readline support
  --enable-debug          enable debugging & verbose explain
  --disable-amalgamation  Disable the amalgamation and instead build all files
                          separately
@@ -1644,7 +1646,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.9.0
+sqlite configure 3.11.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2063,7 +2065,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.9.0, which was
+It was created by sqlite $as_me 3.11.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -3921,13 +3923,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
  lt_cv_nm_interface="BSD nm"
  echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3924: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3926: $ac_compile\"" >&5)
  (eval "$ac_compile" 2>conftest.err)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3927: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3929: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3930: output\"" >&5)
+  (eval echo "\"\$as_me:3932: output\"" >&5)
  cat conftest.out >&5
  if $GREP 'External.*some_variable' conftest.out > /dev/null; then
    lt_cv_nm_interface="MS dumpbin"
@@ -5133,7 +5135,7 @@ ia64-*-hpux*)
  ;;
 *-*-irix6*)
  # Find out which ABI we are using.
-  echo '#line 5136 "configure"' > conftest.$ac_ext
+  echo '#line 5138 "configure"' > conftest.$ac_ext
  if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  (eval $ac_compile) 2>&5
  ac_status=$?
@@ -6658,11 +6660,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6661: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6663: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:6665: \$? = $ac_status" >&5
+   echo "$as_me:6667: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -6997,11 +6999,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7000: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7002: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:7004: \$? = $ac_status" >&5
+   echo "$as_me:7006: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7102,11 +7104,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7105: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7107: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7109: \$? = $ac_status" >&5
+   echo "$as_me:7111: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -7157,11 +7159,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7160: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7162: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7164: \$? = $ac_status" >&5
+   echo "$as_me:7166: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -9537,7 +9539,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9540 "configure"
+#line 9542 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -9633,7 +9635,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9636 "configure"
+#line 9638 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -9953,48 +9955,6 @@ test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'

 test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'

-for ac_prog in gawk mawk nawk awk
-do
-  # Extract the first word of "$ac_prog", so it can be a program name with args.
-set dummy $ac_prog; ac_word=$2
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
-$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_AWK+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-  if test -n "$AWK"; then
-  ac_cv_prog_AWK="$AWK" # Let the user override the test.
-else
-as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
-for as_dir in $PATH
-do
-  IFS=$as_save_IFS
-  test -z "$as_dir" && as_dir=.
-    for ac_exec_ext in '' $ac_executable_extensions; do
-  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
-    ac_cv_prog_AWK="$ac_prog"
-    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
-    break 2
-  fi
-done
-  done
-IFS=$as_save_IFS
-
-fi
-fi
-AWK=$ac_cv_prog_AWK
-if test -n "$AWK"; then
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5
-$as_echo "$AWK" >&6; }
-else
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
-fi
-
-
-  test -n "$AWK" && break
-done
-

 #########
 # Enable large file support (if special flags are necessary)
@@ -10558,6 +10518,62 @@ ac_res=$ac_cv_search_pthread_create
 if test "$ac_res" != no; then :
  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"

+fi
+
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_mutexattr_init" >&5
+$as_echo_n "checking for library containing pthread_mutexattr_init... " >&6; }
+if ${ac_cv_search_pthread_mutexattr_init+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char pthread_mutexattr_init ();
+int
+main ()
+{
+return pthread_mutexattr_init ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' pthread; do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_search_pthread_mutexattr_init=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext
+  if ${ac_cv_search_pthread_mutexattr_init+:} false; then :
+  break
+fi
+done
+if ${ac_cv_search_pthread_mutexattr_init+:} false; then :
+
+else
+  ac_cv_search_pthread_mutexattr_init=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_mutexattr_init" >&5
+$as_echo "$ac_cv_search_pthread_mutexattr_init" >&6; }
+ac_res=$ac_cv_search_pthread_mutexattr_init
+if test "$ac_res" != no; then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
 fi

 fi
@@ -10875,6 +10891,14 @@ fi
 TARGET_READLINE_LIBS=""
 TARGET_READLINE_INC=""
 TARGET_HAVE_READLINE=0
+TARGET_HAVE_EDITLINE=0
+# Check whether --enable-editline was given.
+if test "${enable_editline+set}" = set; then :
+  enableval=$enable_editline; with_editline=$enableval
+else
+  with_editline=auto
+fi
+
 # Check whether --enable-readline was given.
 if test "${enable_readline+set}" = set; then :
  enableval=$enable_readline; with_readline=$enableval
@@ -10883,6 +10907,71 @@ else
 fi


+if test x"$with_editline" != xno; then
+        sLIBS=$LIBS
+        LIBS=""
+	TARGET_HAVE_EDITLINE=1
+	{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing readline" >&5
+$as_echo_n "checking for library containing readline... " >&6; }
+if ${ac_cv_search_readline+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char readline ();
+int
+main ()
+{
+return readline ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' edit; do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_search_readline=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext
+  if ${ac_cv_search_readline+:} false; then :
+  break
+fi
+done
+if ${ac_cv_search_readline+:} false; then :
+
+else
+  ac_cv_search_readline=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_readline" >&5
+$as_echo "$ac_cv_search_readline" >&6; }
+ac_res=$ac_cv_search_readline
+if test "$ac_res" != no; then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+  with_readline=no
+else
+  TARGET_HAVE_EDITLINE=0
+fi
+
+        TARGET_READLINE_LIBS=$LIBS
+        LIBS=$sLIBS
+fi
 if test x"$with_readline" != xno; then
 	found="yes"

@@ -11071,6 +11160,7 @@ fi



+
 ##########
 # Figure out what C libraries are required to compile programs
 # that use "fdatasync()" function.
@@ -11989,7 +12079,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.9.0, which was
+This file was extended by sqlite $as_me 3.11.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12055,7 +12145,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.9.0
+sqlite config.status 3.11.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -12066,7 +12156,6 @@ gives unlimited permission to copy, distribute and modify it."
 ac_pwd='$ac_pwd'
 srcdir='$srcdir'
 INSTALL='$INSTALL'
-AWK='$AWK'
 test -n "\$AWK" || AWK=awk
 _ACEOF

@@ -90,7 +90,6 @@ fi
 #
 AC_PROG_LIBTOOL
 AC_PROG_INSTALL
-AC_PROG_AWK

 #########
 # Enable large file support (if special flags are necessary)
@@ -196,6 +195,7 @@ AC_SUBST(SQLITE_THREADSAFE)

 if test "$SQLITE_THREADSAFE" = "1"; then
  AC_SEARCH_LIBS(pthread_create, pthread)
+  AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
 fi

 ##########
@@ -461,11 +461,24 @@ AC_SUBST(HAVE_TCL)
 TARGET_READLINE_LIBS=""
 TARGET_READLINE_INC=""
 TARGET_HAVE_READLINE=0
+TARGET_HAVE_EDITLINE=0
+AC_ARG_ENABLE([editline],
+	[AC_HELP_STRING([--enable-editline],[enable BSD editline support])],
+	[with_editline=$enableval],
+	[with_editline=auto])
 AC_ARG_ENABLE([readline],
-	[AC_HELP_STRING([--disable-readline],[disable readline support [default=detect]])],
+	[AC_HELP_STRING([--disable-readline],[disable readline support])],
 	[with_readline=$enableval],
 	[with_readline=auto])

+if test x"$with_editline" != xno; then
+        sLIBS=$LIBS
+        LIBS=""
+	TARGET_HAVE_EDITLINE=1
+	AC_SEARCH_LIBS(readline,edit,[with_readline=no],[TARGET_HAVE_EDITLINE=0])
+        TARGET_READLINE_LIBS=$LIBS
+        LIBS=$sLIBS
+fi
 if test x"$with_readline" != xno; then
 	found="yes"

@@ -520,6 +533,7 @@ fi
 AC_SUBST(TARGET_READLINE_LIBS)
 AC_SUBST(TARGET_READLINE_INC)
 AC_SUBST(TARGET_HAVE_READLINE)
+AC_SUBST(TARGET_HAVE_EDITLINE)

 ##########
 # Figure out what C libraries are required to compile programs
@@ -205,13 +205,13 @@ static int getVarint32(const char *p, int *pi){
 */
 /* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
 static int safe_isspace(char c){
-  return (c&0x80)==0 ? isspace(c) : 0;
+  return (c&0x80)==0 ? isspace((unsigned char)c) : 0;
 }
 static int safe_tolower(char c){
-  return (c&0x80)==0 ? tolower(c) : c;
+  return (c&0x80)==0 ? tolower((unsigned char)c) : c;
 }
 static int safe_isalnum(char c){
-  return (c&0x80)==0 ? isalnum(c) : 0;
+  return (c&0x80)==0 ? isalnum((unsigned char)c) : 0;
 }

 typedef enum DocListType {
@@ -138,7 +138,7 @@ static int simpleNext(
        ** case-insensitivity.
        */
        char ch = c->pCurrent[ii];
-        c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch;
+        c->zToken[ii] = (unsigned char)ch<0x80 ? tolower((unsigned char)ch):ch;
      }
      c->zToken[n] = '\0';
      *ppToken = c->zToken;
@@ -398,7 +398,7 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){
    if( sqlite3_step(pStmt)==SQLITE_ROW
     && (nLeaf = sqlite3_column_int(pStmt, 0))>0
    ){
-      int nIdx = sqlite3_column_int(pStmt, 5);
+      nIdx = sqlite3_column_int(pStmt, 5);
      sqlite3_int64 sz;
      printf("For level %d:\n", i);
      printf("  Number of indexes...................... %9d\n", nIdx);
@@ -84,6 +84,9 @@ struct Fts5PhraseIter {
 **   an OOM condition or IO error), an appropriate SQLite error code is 
 **   returned.
 **
+**   This function may be quite inefficient if used with an FTS5 table
+**   created with the "columnsize=0" option.
+**
 ** xColumnText:
 **   This function attempts to retrieve the text of column iCol of the
 **   current document. If successful, (*pz) is set to point to a buffer
@@ -104,15 +107,29 @@ struct Fts5PhraseIter {
 **   the query within the current row. Return SQLITE_OK if successful, or
 **   an error code (i.e. SQLITE_NOMEM) if an error occurs.
 **
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" or "detail=column" option. If the FTS5 table is created 
+**   with either "detail=none" or "detail=column" and "content=" option 
+**   (i.e. if it is a contentless table), then this API always returns 0.
+**
 ** xInst:
 **   Query for the details of phrase match iIdx within the current row.
 **   Phrase matches are numbered starting from zero, so the iIdx argument
 **   should be greater than or equal to zero and smaller than the value
 **   output by xInstCount().
 **
+**   Usually, output parameter *piPhrase is set to the phrase number, *piCol
+**   to the column in which it occurs and *piOff the token offset of the
+**   first token of the phrase. The exception is if the table was created
+**   with the offsets=0 option specified. In this case *piOff is always
+**   set to -1.
+**
 **   Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) 
 **   if an error occurs.
 **
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" or "detail=column" option. 
+**
 ** xRowid:
 **   Returns the rowid of the current row.
 **
@@ -196,7 +213,7 @@ struct Fts5PhraseIter {
 **       Fts5PhraseIter iter;
 **       int iCol, iOff;
 **       for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
-**           iOff>=0;
+**           iCol>=0;
 **           pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
 **       ){
 **         // An instance of phrase iPhrase at offset iOff of column iCol
@@ -204,13 +221,51 @@ struct Fts5PhraseIter {
 **
 **   The Fts5PhraseIter structure is defined above. Applications should not
 **   modify this structure directly - it should only be used as shown above
-**   with the xPhraseFirst() and xPhraseNext() API methods.
+**   with the xPhraseFirst() and xPhraseNext() API methods (and by
+**   xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).
+**
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" or "detail=column" option. If the FTS5 table is created 
+**   with either "detail=none" or "detail=column" and "content=" option 
+**   (i.e. if it is a contentless table), then this API always iterates
+**   through an empty set (all calls to xPhraseFirst() set iCol to -1).
 **
 ** xPhraseNext()
 **   See xPhraseFirst above.
+**
+** xPhraseFirstColumn()
+**   This function and xPhraseNextColumn() are similar to the xPhraseFirst()
+**   and xPhraseNext() APIs described above. The difference is that instead
+**   of iterating through all instances of a phrase in the current row, these
+**   APIs are used to iterate through the set of columns in the current row
+**   that contain one or more instances of a specified phrase. For example:
+**
+**       Fts5PhraseIter iter;
+**       int iCol;
+**       for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
+**           iCol>=0;
+**           pApi->xPhraseNextColumn(pFts, &iter, &iCol)
+**       ){
+**         // Column iCol contains at least one instance of phrase iPhrase
+**       }
+**
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" option. If the FTS5 table is created with either 
+**   "detail=none" "content=" option (i.e. if it is a contentless table), 
+**   then this API always iterates through an empty set (all calls to 
+**   xPhraseFirstColumn() set iCol to -1).
+**
+**   The information accessed using this API and its companion
+**   xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext
+**   (or xInst/xInstCount). The chief advantage of this API is that it is
+**   significantly more efficient than those alternatives when used with
+**   "detail=column" tables.  
+**
+** xPhraseNextColumn()
+**   See xPhraseFirstColumn above.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 1 */
+  int iVersion;                   /* Currently always set to 3 */

  void *(*xUserData)(Fts5Context*);

@@ -240,8 +295,11 @@ struct Fts5ExtensionApi {
  int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
  void *(*xGetAuxdata)(Fts5Context*, int bClear);

-  void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
+  int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
  void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
+
+  int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
+  void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
 };

 /* 
@@ -29,7 +29,7 @@ typedef unsigned short u16;
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;

-#define ArraySize(x) (sizeof(x) / sizeof(x[0]))
+#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))

 #define testcase(x)
 #define ALWAYS(x) 1
@@ -151,6 +151,7 @@ struct Fts5Config {
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
+  int eDetail;                    /* FTS5_DETAIL_XXX value */
  char *zContentExprlist;
  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;
@@ -160,6 +161,7 @@ struct Fts5Config {
  int pgsz;                       /* Approximate page size used in %_data */
  int nAutomerge;                 /* 'automerge' setting */
  int nCrisisMerge;               /* Maximum allowed segments per level */
+  int nHashSize;                  /* Bytes of memory for in-memory hash */
  char *zRank;                    /* Name of rank function */
  char *zRankArgs;                /* Arguments to rank function */

@@ -178,6 +180,9 @@ struct Fts5Config {
 #define FTS5_CONTENT_NONE     1
 #define FTS5_CONTENT_EXTERNAL 2

+#define FTS5_DETAIL_FULL    0
+#define FTS5_DETAIL_NONE    1
+#define FTS5_DETAIL_COLUMNS 2



@@ -224,25 +229,27 @@ struct Fts5Buffer {
  int nSpace;
 };

-int sqlite3Fts5BufferGrow(int*, Fts5Buffer*, int);
+int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
-void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
+void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
 void sqlite3Fts5BufferFree(Fts5Buffer*);
 void sqlite3Fts5BufferZero(Fts5Buffer*);
 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
-void sqlite3Fts5BufferAppend32(int*, Fts5Buffer*, int);

 char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);

 #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
-#define fts5BufferGrow(a,b,c)         sqlite3Fts5BufferGrow(a,b,c)
 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
 #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
 #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
 #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
-#define fts5BufferAppend32(a,b,c)     sqlite3Fts5BufferAppend32(a,b,c)
+
+#define fts5BufferGrow(pRc,pBuf,nn) ( \
+  (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
+    sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
+)

 /* Write and decode big-endian 32-bit integer values */
 void sqlite3Fts5Put32(u8*, int);
@@ -275,6 +282,7 @@ struct Fts5PoslistWriter {
  i64 iPrev;
 };
 int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
+void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);

 int sqlite3Fts5PoslistNext64(
  const u8 *a, int n,             /* Buffer containing poslist */
@@ -289,6 +297,13 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
 /* Character set tests (like isspace(), isalpha() etc.) */
 int sqlite3Fts5IsBareword(char t);

+
+/* Bucket of terms object used by the integrity-check in offsets=0 mode. */
+typedef struct Fts5Termset Fts5Termset;
+int sqlite3Fts5TermsetNew(Fts5Termset**);
+int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
+void sqlite3Fts5TermsetFree(Fts5Termset*);
+
 /*
 ** End of interface to code in fts5_buffer.c.
 **************************************************************************/
@@ -301,6 +316,15 @@ int sqlite3Fts5IsBareword(char t);
 typedef struct Fts5Index Fts5Index;
 typedef struct Fts5IndexIter Fts5IndexIter;

+struct Fts5IndexIter {
+  i64 iRowid;
+  const u8 *pData;
+  int nData;
+  u8 bEof;
+};
+
+#define sqlite3Fts5IterEof(x) ((x)->bEof)
+
 /*
 ** Values used as part of the flags argument passed to IndexQuery().
 */
@@ -309,6 +333,12 @@ typedef struct Fts5IndexIter Fts5IndexIter;
 #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
 #define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */

+/* The following are used internally by the fts5_index.c module. They are
+** defined here only to make it easier to avoid clashes with the flags
+** above. */
+#define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
+#define FTS5INDEX_QUERY_NOOUTPUT   0x0020
+
 /*
 ** Create/destroy an Fts5Index object.
 */
@@ -325,6 +355,29 @@ int sqlite3Fts5IndexClose(Fts5Index *p);
 ** }
 */

+/*
+** Return a simple checksum value based on the arguments.
+*/
+u64 sqlite3Fts5IndexEntryCksum(
+  i64 iRowid, 
+  int iCol, 
+  int iPos, 
+  int iIdx,
+  const char *pTerm,
+  int nTerm
+);
+
+/*
+** Argument p points to a buffer containing utf-8 text that is n bytes in 
+** size. Return the number of bytes in the nChar character prefix of the
+** buffer, or 0 if there are less than nChar characters in total.
+*/
+int sqlite3Fts5IndexCharlenToBytelen(
+  const char *p, 
+  int nByte, 
+  int nChar
+);
+
 /*
 ** Open a new iterator to iterate though all rowids that match the 
 ** specified token or token prefix.
@@ -341,12 +394,9 @@ int sqlite3Fts5IndexQuery(
 ** The various operations on open token or token prefix iterators opened
 ** using sqlite3Fts5IndexQuery().
 */
-int sqlite3Fts5IterEof(Fts5IndexIter*);
 int sqlite3Fts5IterNext(Fts5IndexIter*);
 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
 i64 sqlite3Fts5IterRowid(Fts5IndexIter*);
-int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*);
-int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf);

 /*
 ** Close an iterator opened by sqlite3Fts5IndexQuery().
@@ -410,7 +460,6 @@ int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
 /*
 ** Functions called by the storage module as part of integrity-check.
 */
-u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int);
 int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);

 /* 
@@ -475,7 +524,7 @@ int sqlite3Fts5GetTokenizer(
  char **pzErr
 );

-Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, int*);
+Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **);

 /*
 ** End of interface to code in fts5.c.
@@ -489,7 +538,7 @@ typedef struct Fts5Hash Fts5Hash;
 /*
 ** Create a hash table, free a hash table.
 */
-int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize);
+int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
 void sqlite3Fts5HashFree(Fts5Hash*);

 int sqlite3Fts5HashWrite(
@@ -548,7 +597,7 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 int sqlite3Fts5DropAll(Fts5Config*);
 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);

-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);

@@ -568,8 +617,6 @@ int sqlite3Fts5StorageConfigValue(
    Fts5Storage *p, const char*, sqlite3_value*, int
 );

-int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
-
 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
@@ -626,8 +673,18 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);

+typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
+Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
+int sqlite3Fts5ExprPopulatePoslists(
+    Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
+);
+void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
+void sqlite3Fts5ExprClearEof(Fts5Expr*);
+
 int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);

+int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
+
 /*******************************************
 ** The fts5_expr.c API above this point is used by the other hand-written
 ** C code in this module. The interfaces below this point are called by
@@ -137,7 +137,7 @@ static void fts5HighlightAppend(
  const char *z, int n
 ){
  if( *pRc==SQLITE_OK ){
-    if( n<0 ) n = strlen(z);
+    if( n<0 ) n = (int)strlen(z);
    p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
    if( p->zOut==0 ) *pRc = SQLITE_NOMEM;
  }
@@ -544,7 +544,7 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){
  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */

-  for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
+  for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
    rc = pApi->xCreateFunction(pApi,
        aBuiltin[i].zFunc,
        aBuiltin[i].pUserData,
@@ -15,16 +15,11 @@

 #include "fts5Int.h"

-int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
-
-  if( (pBuf->n + nByte) > pBuf->nSpace ){
+int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
+  if( pBuf->nSpace<nByte ){
+    u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
    u8 *pNew;
-    int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
-
-    /* A no-op if an error has already occurred */
-    if( *pRc ) return 1;
-
-    while( nNew<(pBuf->n + nByte) ){
+    while( nNew<nByte ){
      nNew = nNew * 2;
    }
    pNew = sqlite3_realloc(pBuf->p, nNew);
@@ -39,12 +34,13 @@ int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
  return 0;
 }

+
 /*
 ** Encode value iVal as an SQLite varint and append it to the buffer object
 ** pBuf. If an OOM error occurs, set the error code in p.
 */
 void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
-  if( sqlite3Fts5BufferGrow(pRc, pBuf, 9) ) return;
+  if( fts5BufferGrow(pRc, pBuf, 9) ) return;
  pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal);
 }

@@ -59,12 +55,6 @@ int sqlite3Fts5Get32(const u8 *aBuf){
  return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3];
 }

-void sqlite3Fts5BufferAppend32(int *pRc, Fts5Buffer *pBuf, int iVal){
-  if( sqlite3Fts5BufferGrow(pRc, pBuf, 4) ) return;
-  sqlite3Fts5Put32(&pBuf->p[pBuf->n], iVal);
-  pBuf->n += 4;
-}
-
 /*
 ** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set 
 ** the error code in p. If an error has already occurred when this function
@@ -73,11 +63,11 @@ void sqlite3Fts5BufferAppend32(int *pRc, Fts5Buffer *pBuf, int iVal){
 void sqlite3Fts5BufferAppendBlob(
  int *pRc,
  Fts5Buffer *pBuf, 
-  int nData, 
+  u32 nData, 
  const u8 *pData
 ){
-  assert( *pRc || nData>=0 );
-  if( sqlite3Fts5BufferGrow(pRc, pBuf, nData) ) return;
+  assert_nc( *pRc || nData>=0 );
+  if( fts5BufferGrow(pRc, pBuf, nData) ) return;
  memcpy(&pBuf->p[pBuf->n], pData, nData);
  pBuf->n += nData;
 }
@@ -92,7 +82,7 @@ void sqlite3Fts5BufferAppendString(
  Fts5Buffer *pBuf, 
  const char *zStr
 ){
-  int nStr = strlen(zStr);
+  int nStr = (int)strlen(zStr);
  sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr);
  pBuf->n--;
 }
@@ -220,23 +210,36 @@ int sqlite3Fts5PoslistReaderInit(
  return pIter->bEof;
 }

+/*
+** Append position iPos to the position list being accumulated in buffer
+** pBuf, which must be already be large enough to hold the new data.
+** The previous position written to this list is *piPrev. *piPrev is set
+** to iPos before returning.
+*/
+void sqlite3Fts5PoslistSafeAppend(
+  Fts5Buffer *pBuf, 
+  i64 *piPrev, 
+  i64 iPos
+){
+  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
+  if( (iPos & colmask) != (*piPrev & colmask) ){
+    pBuf->p[pBuf->n++] = 1;
+    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
+    *piPrev = (iPos & colmask);
+  }
+  pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
+  *piPrev = iPos;
+}
+
 int sqlite3Fts5PoslistWriterAppend(
  Fts5Buffer *pBuf, 
  Fts5PoslistWriter *pWriter,
  i64 iPos
 ){
-  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
-  int rc = SQLITE_OK;
-  if( 0==sqlite3Fts5BufferGrow(&rc, pBuf, 5+5+5) ){
-    if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
-      pBuf->p[pBuf->n++] = 1;
-      pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
-      pWriter->iPrev = (iPos & colmask);
-    }
-    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
-    pWriter->iPrev = iPos;
-  }
-  return rc;
+  int rc;
+  if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
+  sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
+  return SQLITE_OK;
 }

 void *sqlite3Fts5MallocZero(int *pRc, int nByte){
@@ -264,7 +267,7 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){
  char *zRet = 0;
  if( *pRc==SQLITE_OK ){
    if( nIn<0 ){
-      nIn = strlen(pIn);
+      nIn = (int)strlen(pIn);
    }
    zRet = (char*)sqlite3_malloc(nIn+1);
    if( zRet ){
@@ -304,3 +307,89 @@ int sqlite3Fts5IsBareword(char t){
 }


+/*************************************************************************
+*/
+typedef struct Fts5TermsetEntry Fts5TermsetEntry;
+struct Fts5TermsetEntry {
+  char *pTerm;
+  int nTerm;
+  int iIdx;                       /* Index (main or aPrefix[] entry) */
+  Fts5TermsetEntry *pNext;
+};
+
+struct Fts5Termset {
+  Fts5TermsetEntry *apHash[512];
+};
+
+int sqlite3Fts5TermsetNew(Fts5Termset **pp){
+  int rc = SQLITE_OK;
+  *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset));
+  return rc;
+}
+
+int sqlite3Fts5TermsetAdd(
+  Fts5Termset *p, 
+  int iIdx,
+  const char *pTerm, int nTerm, 
+  int *pbPresent
+){
+  int rc = SQLITE_OK;
+  *pbPresent = 0;
+  if( p ){
+    int i;
+    u32 hash = 13;
+    Fts5TermsetEntry *pEntry;
+
+    /* Calculate a hash value for this term. This is the same hash checksum
+    ** used by the fts5_hash.c module. This is not important for correct
+    ** operation of the module, but is necessary to ensure that some tests
+    ** designed to produce hash table collisions really do work.  */
+    for(i=nTerm-1; i>=0; i--){
+      hash = (hash << 3) ^ hash ^ pTerm[i];
+    }
+    hash = (hash << 3) ^ hash ^ iIdx;
+    hash = hash % ArraySize(p->apHash);
+
+    for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
+      if( pEntry->iIdx==iIdx 
+          && pEntry->nTerm==nTerm 
+          && memcmp(pEntry->pTerm, pTerm, nTerm)==0 
+      ){
+        *pbPresent = 1;
+        break;
+      }
+    }
+
+    if( pEntry==0 ){
+      pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm);
+      if( pEntry ){
+        pEntry->pTerm = (char*)&pEntry[1];
+        pEntry->nTerm = nTerm;
+        pEntry->iIdx = iIdx;
+        memcpy(pEntry->pTerm, pTerm, nTerm);
+        pEntry->pNext = p->apHash[hash];
+        p->apHash[hash] = pEntry;
+      }
+    }
+  }
+
+  return rc;
+}
+
+void sqlite3Fts5TermsetFree(Fts5Termset *p){
+  if( p ){
+    u32 i;
+    for(i=0; i<ArraySize(p->apHash); i++){
+      Fts5TermsetEntry *pEntry = p->apHash[i];
+      while( pEntry ){
+        Fts5TermsetEntry *pDel = pEntry;
+        pEntry = pEntry->pNext;
+        sqlite3_free(pDel);
+      }
+    }
+    sqlite3_free(p);
+  }
+}
+
+
+
@@ -14,12 +14,12 @@
 */


-
 #include "fts5Int.h"

 #define FTS5_DEFAULT_PAGE_SIZE   4050
 #define FTS5_DEFAULT_AUTOMERGE      4
 #define FTS5_DEFAULT_CRISISMERGE   16
+#define FTS5_DEFAULT_HASHSIZE    (1024*1024)

 /* Maximum allowed page size */
 #define FTS5_MAX_PAGE_SIZE (128*1024)
@@ -194,6 +194,33 @@ void sqlite3Fts5Dequote(char *z){
  }
 }

+
+struct Fts5Enum {
+  const char *zName;
+  int eVal;
+};
+typedef struct Fts5Enum Fts5Enum;
+
+static int fts5ConfigSetEnum(
+  const Fts5Enum *aEnum, 
+  const char *zEnum, 
+  int *peVal
+){
+  int nEnum = strlen(zEnum);
+  int i;
+  int iVal = -1;
+
+  for(i=0; aEnum[i].zName; i++){
+    if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
+      if( iVal>=0 ) return SQLITE_ERROR;
+      iVal = aEnum[i].eVal;
+    }
+  }
+
+  *peVal = iVal;
+  return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
+}
+
 /*
 ** Parse a "special" CREATE VIRTUAL TABLE directive and update
 ** configuration object pConfig as appropriate.
@@ -211,44 +238,63 @@ static int fts5ConfigParseSpecial(
  char **pzErr                    /* OUT: Error message */
 ){
  int rc = SQLITE_OK;
-  int nCmd = strlen(zCmd);
+  int nCmd = (int)strlen(zCmd);
  if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
    const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
    const char *p;
-    if( pConfig->aPrefix ){
-      *pzErr = sqlite3_mprintf("multiple prefix=... directives");
-      rc = SQLITE_ERROR;
-    }else{
+    int bFirst = 1;
+    if( pConfig->aPrefix==0 ){
      pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
+      if( rc ) return rc;
    }
+
    p = zArg;
-    while( rc==SQLITE_OK && p[0] ){
+    while( 1 ){
      int nPre = 0;
+
      while( p[0]==' ' ) p++;
+      if( bFirst==0 && p[0]==',' ){
+        p++;
+        while( p[0]==' ' ) p++;
+      }else if( p[0]=='\0' ){
+        break;
+      }
+      if( p[0]<'0' || p[0]>'9' ){
+        *pzErr = sqlite3_mprintf("malformed prefix=... directive");
+        rc = SQLITE_ERROR;
+        break;
+      }
+
+      if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){
+        *pzErr = sqlite3_mprintf(
+            "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES
+        );
+        rc = SQLITE_ERROR;
+        break;
+      }
+
      while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
        nPre = nPre*10 + (p[0] - '0');
        p++;
      }
-      while( p[0]==' ' ) p++;
-      if( p[0]==',' ){
-        p++;
-      }else if( p[0] ){
-        *pzErr = sqlite3_mprintf("malformed prefix=... directive");
-        rc = SQLITE_ERROR;
-      }
-      if( rc==SQLITE_OK && (nPre==0 || nPre>=1000) ){
-        *pzErr = sqlite3_mprintf("prefix length out of range: %d", nPre);
+
+      if( nPre<=0 || nPre>=1000 ){
+        *pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
        rc = SQLITE_ERROR;
+        break;
      }
+
      pConfig->aPrefix[pConfig->nPrefix] = nPre;
      pConfig->nPrefix++;
+      bFirst = 0;
    }
+    assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES );
    return rc;
  }

  if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
    const char *p = (const char*)zArg;
-    int nArg = strlen(zArg) + 1;
+    int nArg = (int)strlen(zArg) + 1;
    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
    char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
    char *pSpace = pDel;
@@ -325,6 +371,20 @@ static int fts5ConfigParseSpecial(
    return rc;
  }

+  if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
+    const Fts5Enum aDetail[] = {
+      { "none", FTS5_DETAIL_NONE },
+      { "full", FTS5_DETAIL_FULL },
+      { "columns", FTS5_DETAIL_COLUMNS },
+      { 0, 0 }
+    };
+
+    if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){
+      *pzErr = sqlite3_mprintf("malformed detail=... directive");
+    }
+    return rc;
+  }
+
  *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
  return SQLITE_ERROR;
 }
@@ -364,7 +424,7 @@ static const char *fts5ConfigGobbleWord(
 ){
  const char *zRet = 0;

-  int nIn = strlen(zIn);
+  int nIn = (int)strlen(zIn);
  char *zOut = sqlite3_malloc(nIn+1);

  assert( *pRc==SQLITE_OK );
@@ -480,6 +540,7 @@ int sqlite3Fts5ConfigParse(
  pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
  pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
  pRet->bColumnsize = 1;
+  pRet->eDetail = FTS5_DETAIL_FULL;
 #ifdef SQLITE_DEBUG
  pRet->bPrefixIndex = 1;
 #endif
@@ -706,33 +767,37 @@ int sqlite3Fts5ConfigParseRank(
  *pzRank = 0;
  *pzRankArgs = 0;

-  p = fts5ConfigSkipWhitespace(p);
-  pRank = p;
-  p = fts5ConfigSkipBareword(p);
-
-  if( p ){
-    zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
-    if( zRank ) memcpy(zRank, pRank, p-pRank);
-  }else{
+  if( p==0 ){
    rc = SQLITE_ERROR;
-  }
+  }else{
+    p = fts5ConfigSkipWhitespace(p);
+    pRank = p;
+    p = fts5ConfigSkipBareword(p);

-  if( rc==SQLITE_OK ){
-    p = fts5ConfigSkipWhitespace(p);
-    if( *p!='(' ) rc = SQLITE_ERROR;
-    p++;
-  }
-  if( rc==SQLITE_OK ){
-    const char *pArgs; 
-    p = fts5ConfigSkipWhitespace(p);
-    pArgs = p;
-    if( *p!=')' ){
-      p = fts5ConfigSkipArgs(p);
-      if( p==0 ){
-        rc = SQLITE_ERROR;
-      }else{
-        zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
-        if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
+    if( p ){
+      zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
+      if( zRank ) memcpy(zRank, pRank, p-pRank);
+    }else{
+      rc = SQLITE_ERROR;
+    }
+
+    if( rc==SQLITE_OK ){
+      p = fts5ConfigSkipWhitespace(p);
+      if( *p!='(' ) rc = SQLITE_ERROR;
+      p++;
+    }
+    if( rc==SQLITE_OK ){
+      const char *pArgs; 
+      p = fts5ConfigSkipWhitespace(p);
+      pArgs = p;
+      if( *p!=')' ){
+        p = fts5ConfigSkipArgs(p);
+        if( p==0 ){
+          rc = SQLITE_ERROR;
+        }else{
+          zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
+          if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
+        }
      }
    }
  }
@@ -767,6 +832,18 @@ int sqlite3Fts5ConfigSetValue(
    }
  }

+  else if( 0==sqlite3_stricmp(zKey, "hashsize") ){
+    int nHashSize = -1;
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      nHashSize = sqlite3_value_int(pVal);
+    }
+    if( nHashSize<=0 ){
+      *pbBadkey = 1;
+    }else{
+      pConfig->nHashSize = nHashSize;
+    }
+  }
+
  else if( 0==sqlite3_stricmp(zKey, "automerge") ){
    int nAutomerge = -1;
    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
@@ -827,6 +904,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
  pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
  pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
  pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
+  pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;

  zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
  if( zSql ){
@@ -26,6 +26,7 @@ typedef struct Fts5HashEntry Fts5HashEntry;


 struct Fts5Hash {
+  int eDetail;                    /* Copy of Fts5Config.eDetail */
  int *pnByte;                    /* Pointer to bytes counter */
  int nEntry;                     /* Number of entries currently in hash */
  int nSlot;                      /* Size of aSlot[] array */
@@ -62,6 +63,7 @@ struct Fts5HashEntry {
  int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
  int nData;                      /* Total bytes of data (incl. structure) */
  u8 bDel;                        /* Set delete-flag @ iSzPoslist */
+  u8 bContent;                    /* Set content-flag (detail=none mode) */

  int iCol;                       /* Column of last value written */
  int iPos;                       /* Position of last value written */
@@ -79,7 +81,7 @@ struct Fts5HashEntry {
 /*
 ** Allocate a new hash table.
 */
-int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){
+int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
  int rc = SQLITE_OK;
  Fts5Hash *pNew;

@@ -90,6 +92,7 @@ int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){
    int nByte;
    memset(pNew, 0, sizeof(Fts5Hash));
    pNew->pnByte = pnByte;
+    pNew->eDetail = pConfig->eDetail;

    pNew->nSlot = 1024;
    nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
@@ -170,7 +173,7 @@ static int fts5HashResize(Fts5Hash *pHash){
      int iHash;
      Fts5HashEntry *p = apOld[i];
      apOld[i] = p->pHashNext;
-      iHash = fts5HashKey(nNew, (u8*)p->zKey, strlen(p->zKey));
+      iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey));
      p->pHashNext = apNew[iHash];
      apNew[iHash] = p;
    }
@@ -182,26 +185,46 @@ static int fts5HashResize(Fts5Hash *pHash){
  return SQLITE_OK;
 }

-static void fts5HashAddPoslistSize(Fts5HashEntry *p){
+static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
  if( p->iSzPoslist ){
    u8 *pPtr = (u8*)p;
-    int nSz = (p->nData - p->iSzPoslist - 1);         /* Size in bytes */
-    int nPos = nSz*2 + p->bDel;                       /* Value of nPos field */
-
-    assert( p->bDel==0 || p->bDel==1 );
-    if( nPos<=127 ){
-      pPtr[p->iSzPoslist] = nPos;
+    if( pHash->eDetail==FTS5_DETAIL_NONE ){
+      assert( p->nData==p->iSzPoslist );
+      if( p->bDel ){
+        pPtr[p->nData++] = 0x00;
+        if( p->bContent ){
+          pPtr[p->nData++] = 0x00;
+        }
+      }
    }else{
-      int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
-      memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
-      sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
-      p->nData += (nByte-1);
+      int nSz = (p->nData - p->iSzPoslist - 1);       /* Size in bytes */
+      int nPos = nSz*2 + p->bDel;                     /* Value of nPos field */
+
+      assert( p->bDel==0 || p->bDel==1 );
+      if( nPos<=127 ){
+        pPtr[p->iSzPoslist] = (u8)nPos;
+      }else{
+        int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
+        memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
+        sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
+        p->nData += (nByte-1);
+      }
    }
-    p->bDel = 0;
+
    p->iSzPoslist = 0;
+    p->bDel = 0;
+    p->bContent = 0;
  }
 }

+/*
+** Add an entry to the in-memory hash table. The key is the concatenation
+** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
+**
+**     (bByte || pToken) -> (iRowid,iCol,iPos)
+**
+** Or, if iCol is negative, then the value is a delete marker.
+*/
 int sqlite3Fts5HashWrite(
  Fts5Hash *pHash,
  i64 iRowid,                     /* Rowid for this entry */
@@ -214,6 +237,9 @@ int sqlite3Fts5HashWrite(
  Fts5HashEntry *p;
  u8 *pPtr;
  int nIncr = 0;                  /* Amount to increment (*pHash->pnByte) by */
+  int bNew;                       /* If non-delete entry should be written */
+  
+  bNew = (pHash->eDetail==FTS5_DETAIL_FULL);

  /* Attempt to locate an existing hash entry */
  iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
@@ -228,15 +254,18 @@ int sqlite3Fts5HashWrite(

  /* If an existing hash entry cannot be found, create a new one. */
  if( p==0 ){
+    /* Figure out how much space to allocate */
    int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
    if( nByte<128 ) nByte = 128;

+    /* Grow the Fts5Hash.aSlot[] array if necessary. */
    if( (pHash->nEntry*2)>=pHash->nSlot ){
      int rc = fts5HashResize(pHash);
      if( rc!=SQLITE_OK ) return rc;
      iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
    }

+    /* Allocate new Fts5HashEntry and add it to the hash table. */
    p = (Fts5HashEntry*)sqlite3_malloc(nByte);
    if( !p ) return SQLITE_NOMEM;
    memset(p, 0, FTS5_HASHENTRYSIZE);
@@ -246,70 +275,95 @@ int sqlite3Fts5HashWrite(
    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
    p->zKey[nToken+1] = '\0';
    p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
-    p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
-    p->iSzPoslist = p->nData;
-    p->nData += 1;
-    p->iRowid = iRowid;
    p->pHashNext = pHash->aSlot[iHash];
    pHash->aSlot[iHash] = p;
    pHash->nEntry++;
-    nIncr += p->nData;
-  }

-  /* Check there is enough space to append a new entry. Worst case scenario
-  ** is:
-  **
-  **     + 9 bytes for a new rowid,
-  **     + 4 byte reserved for the "poslist size" varint.
-  **     + 1 byte for a "new column" byte,
-  **     + 3 bytes for a new column number (16-bit max) as a varint,
-  **     + 5 bytes for the new position offset (32-bit max).
-  */
-  if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
-    int nNew = p->nAlloc * 2;
-    Fts5HashEntry *pNew;
-    Fts5HashEntry **pp;
-    pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
-    if( pNew==0 ) return SQLITE_NOMEM;
-    pNew->nAlloc = nNew;
-    for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
-    *pp = pNew;
-    p = pNew;
+    /* Add the first rowid field to the hash-entry */
+    p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
+    p->iRowid = iRowid;
+
+    p->iSzPoslist = p->nData;
+    if( pHash->eDetail!=FTS5_DETAIL_NONE ){
+      p->nData += 1;
+      p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
+    }
+
+    nIncr += p->nData;
+  }else{
+
+    /* Appending to an existing hash-entry. Check that there is enough 
+    ** space to append the largest possible new entry. Worst case scenario 
+    ** is:
+    **
+    **     + 9 bytes for a new rowid,
+    **     + 4 byte reserved for the "poslist size" varint.
+    **     + 1 byte for a "new column" byte,
+    **     + 3 bytes for a new column number (16-bit max) as a varint,
+    **     + 5 bytes for the new position offset (32-bit max).
+    */
+    if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
+      int nNew = p->nAlloc * 2;
+      Fts5HashEntry *pNew;
+      Fts5HashEntry **pp;
+      pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
+      if( pNew==0 ) return SQLITE_NOMEM;
+      pNew->nAlloc = nNew;
+      for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
+      *pp = pNew;
+      p = pNew;
+    }
+    nIncr -= p->nData;
  }
+  assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) );
+
  pPtr = (u8*)p;
-  nIncr -= p->nData;

  /* If this is a new rowid, append the 4-byte size field for the previous
  ** entry, and the new rowid for this entry.  */
  if( iRowid!=p->iRowid ){
-    fts5HashAddPoslistSize(p);
+    fts5HashAddPoslistSize(pHash, p);
    p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
-    p->iSzPoslist = p->nData;
-    p->nData += 1;
-    p->iCol = 0;
-    p->iPos = 0;
    p->iRowid = iRowid;
+    bNew = 1;
+    p->iSzPoslist = p->nData;
+    if( pHash->eDetail!=FTS5_DETAIL_NONE ){
+      p->nData += 1;
+      p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
+      p->iPos = 0;
+    }
  }

  if( iCol>=0 ){
-    /* Append a new column value, if necessary */
-    assert( iCol>=p->iCol );
-    if( iCol!=p->iCol ){
-      pPtr[p->nData++] = 0x01;
-      p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
-      p->iCol = iCol;
-      p->iPos = 0;
-    }
+    if( pHash->eDetail==FTS5_DETAIL_NONE ){
+      p->bContent = 1;
+    }else{
+      /* Append a new column value, if necessary */
+      assert( iCol>=p->iCol );
+      if( iCol!=p->iCol ){
+        if( pHash->eDetail==FTS5_DETAIL_FULL ){
+          pPtr[p->nData++] = 0x01;
+          p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
+          p->iCol = iCol;
+          p->iPos = 0;
+        }else{
+          bNew = 1;
+          p->iCol = iPos = iCol;
+        }
+      }

-    /* Append the new position offset */
-    p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
-    p->iPos = iPos;
+      /* Append the new position offset, if necessary */
+      if( bNew ){
+        p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
+        p->iPos = iPos;
+      }
+    }
  }else{
    /* This is a delete. Set the delete flag. */
    p->bDel = 1;
  }
-  nIncr += p->nData;

+  nIncr += p->nData;
  *pHash->pnByte += nIncr;
  return SQLITE_OK;
 }
@@ -423,7 +477,7 @@ int sqlite3Fts5HashQuery(
  }

  if( p ){
-    fts5HashAddPoslistSize(p);
+    fts5HashAddPoslistSize(pHash, p);
    *ppDoclist = (const u8*)&p->zKey[nTerm+1];
    *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
  }else{
@@ -458,8 +512,8 @@ void sqlite3Fts5HashScanEntry(
 ){
  Fts5HashEntry *p;
  if( (p = pHash->pScan) ){
-    int nTerm = strlen(p->zKey);
-    fts5HashAddPoslistSize(p);
+    int nTerm = (int)strlen(p->zKey);
+    fts5HashAddPoslistSize(pHash, p);
    *pzTerm = p->zKey;
    *ppDoclist = (const u8*)&p->zKey[nTerm+1];
    *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
@@ -220,12 +220,13 @@ struct Fts5Cursor {
 /*
 ** Values for Fts5Cursor.csrflags
 */
-#define FTS5CSR_REQUIRE_CONTENT   0x01
-#define FTS5CSR_REQUIRE_DOCSIZE   0x02
-#define FTS5CSR_REQUIRE_INST      0x04
-#define FTS5CSR_EOF               0x08
+#define FTS5CSR_EOF               0x01
+#define FTS5CSR_REQUIRE_CONTENT   0x02
+#define FTS5CSR_REQUIRE_DOCSIZE   0x04
+#define FTS5CSR_REQUIRE_INST      0x08
 #define FTS5CSR_FREE_ZRANK        0x10
 #define FTS5CSR_REQUIRE_RESEEK    0x20
+#define FTS5CSR_REQUIRE_POSLIST   0x40

 #define BitFlagAllTest(x,y) (((x) & (y))==(y))
 #define BitFlagTest(x,y)    (((x) & (y))!=0)
@@ -395,6 +396,15 @@ static int fts5InitVtab(
    rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
  }

+  /* Load the initial configuration */
+  if( rc==SQLITE_OK ){
+    assert( pConfig->pzErrmsg==0 );
+    pConfig->pzErrmsg = pzErr;
+    rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex);
+    sqlite3Fts5IndexRollback(pTab->pIndex);
+    pConfig->pzErrmsg = 0;
+  }
+
  if( rc!=SQLITE_OK ){
    fts5FreeVtab(pTab);
    pTab = 0;
@@ -447,7 +457,10 @@ static int fts5CreateMethod(
 */
 static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
 #if SQLITE_VERSION_NUMBER>=3008012
-  if( sqlite3_libversion_number()>=3008012 ){
+#ifndef SQLITE_CORE
+  if( sqlite3_libversion_number()>=3008012 )
+#endif
+  {
    pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE;
  }
 #endif
@@ -525,7 +538,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
    int j;
-    for(j=0; j<sizeof(aConstraint)/sizeof(aConstraint[0]); j++){
+    for(j=0; j<ArraySize(aConstraint); j++){
      struct Constraint *pC = &aConstraint[j];
      if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){
        if( p->usable ){
@@ -572,11 +585,11 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){

  /* Assign argvIndex values to each constraint in use. */
  iNext = 1;
-  for(i=0; i<sizeof(aConstraint)/sizeof(aConstraint[0]); i++){
+  for(i=0; i<ArraySize(aConstraint); i++){
    struct Constraint *pC = &aConstraint[i];
    if( pC->iConsIndex>=0 ){
      pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++;
-      pInfo->aConstraintUsage[pC->iConsIndex].omit = pC->omit;
+      pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit;
    }
  }

@@ -627,6 +640,7 @@ static void fts5CsrNewrow(Fts5Cursor *pCsr){
      FTS5CSR_REQUIRE_CONTENT 
    | FTS5CSR_REQUIRE_DOCSIZE 
    | FTS5CSR_REQUIRE_INST 
+    | FTS5CSR_REQUIRE_POSLIST 
  );
 }

@@ -709,15 +723,18 @@ static int fts5SorterNext(Fts5Cursor *pCsr){
    nBlob = sqlite3_column_bytes(pSorter->pStmt, 1);
    aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1);

-    for(i=0; i<(pSorter->nIdx-1); i++){
-      int iVal;
-      a += fts5GetVarint32(a, iVal);
-      iOff += iVal;
-      pSorter->aIdx[i] = iOff;
+    /* nBlob==0 in detail=none mode. */
+    if( nBlob>0 ){
+      for(i=0; i<(pSorter->nIdx-1); i++){
+        int iVal;
+        a += fts5GetVarint32(a, iVal);
+        iOff += iVal;
+        pSorter->aIdx[i] = iOff;
+      }
+      pSorter->aIdx[i] = &aBlob[nBlob] - a;
+      pSorter->aPoslist = a;
    }
-    pSorter->aIdx[i] = &aBlob[nBlob] - a;

-    pSorter->aPoslist = a;
    fts5CsrNewrow(pCsr);
  }

@@ -761,7 +778,7 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
    i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);

    rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, iRowid, bDesc);
-    if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){
+    if( rc==SQLITE_OK &&  iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){
      *pbSkip = 1;
    }

@@ -769,6 +786,7 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
    fts5CsrNewrow(pCsr);
    if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
      CsrFlagSet(pCsr, FTS5CSR_EOF);
+      *pbSkip = 1;
    }
  }
  return rc;
@@ -785,24 +803,24 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
 */
 static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
-  int rc = SQLITE_OK;
+  int rc;

  assert( (pCsr->ePlan<3)==
          (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) 
  );
+  assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );

  if( pCsr->ePlan<3 ){
    int bSkip = 0;
    if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
    rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
-    if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
-      CsrFlagSet(pCsr, FTS5CSR_EOF);
-    }
+    CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr));
    fts5CsrNewrow(pCsr);
  }else{
    switch( pCsr->ePlan ){
      case FTS5_PLAN_SPECIAL: {
        CsrFlagSet(pCsr, FTS5CSR_EOF);
+        rc = SQLITE_OK;
        break;
      }
  
@@ -826,13 +844,41 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
  return rc;
 }

+
+static int fts5PrepareStatement(
+  sqlite3_stmt **ppStmt,
+  Fts5Config *pConfig, 
+  const char *zFmt,
+  ...
+){
+  sqlite3_stmt *pRet = 0;
+  int rc;
+  char *zSql;
+  va_list ap;
+
+  va_start(ap, zFmt);
+  zSql = sqlite3_vmprintf(zFmt, ap);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM; 
+  }else{
+    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
+    if( rc!=SQLITE_OK ){
+      *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
+    }
+    sqlite3_free(zSql);
+  }
+
+  va_end(ap);
+  *ppStmt = pRet;
+  return rc;
+} 
+
 static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
  Fts5Config *pConfig = pTab->pConfig;
  Fts5Sorter *pSorter;
  int nPhrase;
  int nByte;
-  int rc = SQLITE_OK;
-  char *zSql;
+  int rc;
  const char *zRank = pCsr->zRank;
  const char *zRankArgs = pCsr->zRankArgs;
  
@@ -850,17 +896,13 @@ static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
  ** table, saving it creates a circular reference.
  **
  ** If SQLite a built-in statement cache, this wouldn't be a problem. */
-  zSql = sqlite3Fts5Mprintf(&rc, 
+  rc = fts5PrepareStatement(&pSorter->pStmt, pConfig,
      "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s",
      pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
      (zRankArgs ? ", " : ""),
      (zRankArgs ? zRankArgs : ""),
      bDesc ? "DESC" : "ASC"
  );
-  if( zSql ){
-    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0);
-    sqlite3_free(zSql);
-  }

  pCsr->pSorter = pSorter;
  if( rc==SQLITE_OK ){
@@ -1130,6 +1172,7 @@ static int fts5FilterMethod(
    pCsr->ePlan = FTS5_PLAN_SOURCE;
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bDesc);
+    sqlite3Fts5ExprClearEof(pCsr->pExpr);
  }else if( pMatch ){
    const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
    if( zExpr==0 ) zExpr = "";
@@ -1362,7 +1405,7 @@ static int fts5SpecialDelete(
  int eType1 = sqlite3_value_type(apVal[1]);
  if( eType1==SQLITE_INTEGER ){
    sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
-    rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
+    rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]);
  }
  return rc;
 }
@@ -1448,7 +1491,10 @@ static int fts5UpdateMethod(
    **
    ** Cases 3 and 4 may violate the rowid constraint.
    */
-    int eConflict = sqlite3_vtab_on_conflict(pConfig->db);
+    int eConflict = SQLITE_ABORT;
+    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
+      eConflict = sqlite3_vtab_on_conflict(pConfig->db);
+    }

    assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
    assert( nArg!=1 || eType0==SQLITE_INTEGER );
@@ -1466,7 +1512,7 @@ static int fts5UpdateMethod(
    /* Case 1: DELETE */
    else if( nArg==1 ){
      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
-      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
+      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
    }

    /* Case 2: INSERT */
@@ -1476,7 +1522,7 @@ static int fts5UpdateMethod(
       && sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
      ){
        i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
-        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
+        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
      }
      fts5StorageInsert(&rc, pTab, apVal, pRowid);
    }
@@ -1487,22 +1533,22 @@ static int fts5UpdateMethod(
      i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
      if( iOld!=iNew ){
        if( eConflict==SQLITE_REPLACE ){
-          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
+          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
          if( rc==SQLITE_OK ){
-            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
+            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
          }
          fts5StorageInsert(&rc, pTab, apVal, pRowid);
        }else{
          rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid);
          if( rc==SQLITE_OK ){
-            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
+            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
          }
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid);
          }
        }
      }else{
-        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
+        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
        fts5StorageInsert(&rc, pTab, apVal, pRowid);
      }
    }
@@ -1556,6 +1602,8 @@ static int fts5RollbackMethod(sqlite3_vtab *pVtab){
  return rc;
 }

+static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*);
+
 static void *fts5ApiUserData(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return pCsr->pAux->pUserData;
@@ -1605,17 +1653,72 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
  return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
 }

-static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){
-  int n;
-  if( pCsr->pSorter ){
+static int fts5ApiColumnText(
+  Fts5Context *pCtx, 
+  int iCol, 
+  const char **pz, 
+  int *pn
+){
+  int rc = SQLITE_OK;
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
+    *pz = 0;
+    *pn = 0;
+  }else{
+    rc = fts5SeekCursor(pCsr, 0);
+    if( rc==SQLITE_OK ){
+      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
+      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
+    }
+  }
+  return rc;
+}
+
+static int fts5CsrPoslist(
+  Fts5Cursor *pCsr, 
+  int iPhrase, 
+  const u8 **pa,
+  int *pn
+){
+  Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
+  int rc = SQLITE_OK;
+  int bLive = (pCsr->pSorter==0);
+
+  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
+
+    if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
+      Fts5PoslistPopulator *aPopulator;
+      int i;
+      aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive);
+      if( aPopulator==0 ) rc = SQLITE_NOMEM;
+      for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
+        int n; const char *z;
+        rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n);
+        if( rc==SQLITE_OK ){
+          rc = sqlite3Fts5ExprPopulatePoslists(
+              pConfig, pCsr->pExpr, aPopulator, i, z, n
+          );
+        }
+      }
+      sqlite3_free(aPopulator);
+
+      if( pCsr->pSorter ){
+        sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid);
+      }
+    }
+    CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST);
+  }
+
+  if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){
    Fts5Sorter *pSorter = pCsr->pSorter;
    int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
-    n = pSorter->aIdx[iPhrase] - i1;
+    *pn = pSorter->aIdx[iPhrase] - i1;
    *pa = &pSorter->aPoslist[i1];
  }else{
-    n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
+    *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
  }
-  return n;
+
+  return rc;
 }

 /*
@@ -1640,43 +1743,48 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){
    int i;

    /* Initialize all iterators */
-    for(i=0; i<nIter; i++){
+    for(i=0; i<nIter && rc==SQLITE_OK; i++){
      const u8 *a;
-      int n = fts5CsrPoslist(pCsr, i, &a);
-      sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
+      int n; 
+      rc = fts5CsrPoslist(pCsr, i, &a, &n);
+      if( rc==SQLITE_OK ){
+        sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
+      }
    }

-    while( 1 ){
-      int *aInst;
-      int iBest = -1;
-      for(i=0; i<nIter; i++){
-        if( (aIter[i].bEof==0) 
-         && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 
-        ){
-          iBest = i;
+    if( rc==SQLITE_OK ){
+      while( 1 ){
+        int *aInst;
+        int iBest = -1;
+        for(i=0; i<nIter; i++){
+          if( (aIter[i].bEof==0) 
+              && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 
+            ){
+            iBest = i;
+          }
        }
-      }
-      if( iBest<0 ) break;
+        if( iBest<0 ) break;

-      nInst++;
-      if( nInst>=pCsr->nInstAlloc ){
-        pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
-        aInst = (int*)sqlite3_realloc(
-            pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3
-        );
-        if( aInst ){
-          pCsr->aInst = aInst;
-        }else{
-          rc = SQLITE_NOMEM;
-          break;
+        nInst++;
+        if( nInst>=pCsr->nInstAlloc ){
+          pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
+          aInst = (int*)sqlite3_realloc(
+              pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3
+              );
+          if( aInst ){
+            pCsr->aInst = aInst;
+          }else{
+            rc = SQLITE_NOMEM;
+            break;
+          }
        }
-      }

-      aInst = &pCsr->aInst[3 * (nInst-1)];
-      aInst[0] = iBest;
-      aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos);
-      aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos);
-      sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
+        aInst = &pCsr->aInst[3 * (nInst-1)];
+        aInst[0] = iBest;
+        aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos);
+        aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos);
+        sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
+      }
    }

    pCsr->nInstCount = nInst;
@@ -1709,6 +1817,12 @@ static int fts5ApiInst(
  ){
    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
      rc = SQLITE_RANGE;
+#if 0
+    }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
+      *piPhrase = pCsr->aInst[iIdx*3];
+      *piCol = pCsr->aInst[iIdx*3 + 2];
+      *piOff = -1;
+#endif
    }else{
      *piPhrase = pCsr->aInst[iIdx*3];
      *piCol = pCsr->aInst[iIdx*3 + 1];
@@ -1722,27 +1836,6 @@ static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
  return fts5CursorRowid((Fts5Cursor*)pCtx);
 }

-static int fts5ApiColumnText(
-  Fts5Context *pCtx, 
-  int iCol, 
-  const char **pz, 
-  int *pn
-){
-  int rc = SQLITE_OK;
-  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
-  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
-    *pz = 0;
-    *pn = 0;
-  }else{
-    rc = fts5SeekCursor(pCsr, 0);
-    if( rc==SQLITE_OK ){
-      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
-      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
-    }
-  }
-  return rc;
-}
-
 static int fts5ColumnSizeCb(
  void *pContext,                 /* Pointer to int */
  int tflags,
@@ -1887,20 +1980,98 @@ static void fts5ApiPhraseNext(
  }
 }

-static void fts5ApiPhraseFirst(
+static int fts5ApiPhraseFirst(
  Fts5Context *pCtx, 
  int iPhrase, 
  Fts5PhraseIter *pIter, 
  int *piCol, int *piOff
 ){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
-  int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a);
-  pIter->b = &pIter->a[n];
-  *piCol = 0;
-  *piOff = 0;
-  fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
+  int n;
+  int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
+  if( rc==SQLITE_OK ){
+    pIter->b = &pIter->a[n];
+    *piCol = 0;
+    *piOff = 0;
+    fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
+  }
+  return rc;
 }

+static void fts5ApiPhraseNextColumn(
+  Fts5Context *pCtx, 
+  Fts5PhraseIter *pIter, 
+  int *piCol
+){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
+
+  if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+    if( pIter->a>=pIter->b ){
+      *piCol = -1;
+    }else{
+      int iIncr;
+      pIter->a += fts5GetVarint32(&pIter->a[0], iIncr);
+      *piCol += (iIncr-2);
+    }
+  }else{
+    while( 1 ){
+      int dummy;
+      if( pIter->a>=pIter->b ){
+        *piCol = -1;
+        return;
+      }
+      if( pIter->a[0]==0x01 ) break;
+      pIter->a += fts5GetVarint32(pIter->a, dummy);
+    }
+    pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
+  }
+}
+
+static int fts5ApiPhraseFirstColumn(
+  Fts5Context *pCtx, 
+  int iPhrase, 
+  Fts5PhraseIter *pIter, 
+  int *piCol
+){
+  int rc = SQLITE_OK;
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
+
+  if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+    Fts5Sorter *pSorter = pCsr->pSorter;
+    int n;
+    if( pSorter ){
+      int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
+      n = pSorter->aIdx[iPhrase] - i1;
+      pIter->a = &pSorter->aPoslist[i1];
+    }else{
+      rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
+    }
+    if( rc==SQLITE_OK ){
+      pIter->b = &pIter->a[n];
+      *piCol = 0;
+      fts5ApiPhraseNextColumn(pCtx, pIter, piCol);
+    }
+  }else{
+    int n;
+    rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
+    if( rc==SQLITE_OK ){
+      pIter->b = &pIter->a[n];
+      if( n<=0 ){
+        *piCol = -1;
+      }else if( pIter->a[0]==0x01 ){
+        pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
+      }else{
+        *piCol = 0;
+      }
+    }
+  }
+
+  return rc;
+}
+
+
 static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
    int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
 );
@@ -1924,9 +2095,10 @@ static const Fts5ExtensionApi sFts5Api = {
  fts5ApiGetAuxdata,
  fts5ApiPhraseFirst,
  fts5ApiPhraseNext,
+  fts5ApiPhraseFirstColumn,
+  fts5ApiPhraseNextColumn,
 };

-
 /*
 ** Implementation of API function xQueryPhrase().
 */
@@ -2018,20 +2190,20 @@ static void fts5ApiCallback(
 ** Given cursor id iId, return a pointer to the corresponding Fts5Index 
 ** object. Or NULL If the cursor id does not exist.
 **
-** If successful, set *pnCol to the number of indexed columns in the
-** table before returning.
+** If successful, set *ppConfig to point to the associated config object 
+** before returning.
 */
 Fts5Index *sqlite3Fts5IndexFromCsrid(
-  Fts5Global *pGlobal, 
-  i64 iCsrId, 
-  int *pnCol
+  Fts5Global *pGlobal,            /* FTS5 global context for db handle */
+  i64 iCsrId,                     /* Id of cursor to find */
+  Fts5Config **ppConfig           /* OUT: Configuration object */
 ){
  Fts5Cursor *pCsr;
  Fts5Table *pTab;

  pCsr = fts5CursorFromCsrid(pGlobal, iCsrId);
  pTab = (Fts5Table*)pCsr->base.pVtab;
-  *pnCol = pTab->pConfig->nCol;
+  *ppConfig = pTab->pConfig;

  return pTab->pIndex;
 }
@@ -2058,20 +2230,46 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
  Fts5Buffer val;

  memset(&val, 0, sizeof(Fts5Buffer));
+  switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){
+    case FTS5_DETAIL_FULL:

-  /* Append the varints */
-  for(i=0; i<(nPhrase-1); i++){
-    const u8 *dummy;
-    int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
-    sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
-  }
+      /* Append the varints */
+      for(i=0; i<(nPhrase-1); i++){
+        const u8 *dummy;
+        int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
+        sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
+      }

-  /* Append the position lists */
-  for(i=0; i<nPhrase; i++){
-    const u8 *pPoslist;
-    int nPoslist;
-    nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
-    sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
+      /* Append the position lists */
+      for(i=0; i<nPhrase; i++){
+        const u8 *pPoslist;
+        int nPoslist;
+        nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
+        sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
+      }
+      break;
+
+    case FTS5_DETAIL_COLUMNS:
+
+      /* Append the varints */
+      for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){
+        const u8 *dummy;
+        int nByte;
+        rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte);
+        sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
+      }
+
+      /* Append the position lists */
+      for(i=0; rc==SQLITE_OK && i<nPhrase; i++){
+        const u8 *pPoslist;
+        int nPoslist;
+        rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist);
+        sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
+      }
+      break;
+
+    default:
+      break;
  }

  sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free);
@@ -297,10 +297,10 @@ int sqlite3Fts5StorageOpen(
        int i;
        int iOff;
        sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY");
-        iOff = strlen(zDefn);
+        iOff = (int)strlen(zDefn);
        for(i=0; i<pConfig->nCol; i++){
          sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
-          iOff += strlen(&zDefn[iOff]);
+          iOff += (int)strlen(&zDefn[iOff]);
        }
        rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
      }
@@ -378,39 +378,52 @@ static int fts5StorageInsertCallback(
 ** delete-markers to the FTS index necessary to delete it. Do not actually
 ** remove the %_content row at this time though.
 */
-static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
+static int fts5StorageDeleteFromIndex(
+  Fts5Storage *p, 
+  i64 iDel, 
+  sqlite3_value **apVal
+){
  Fts5Config *pConfig = p->pConfig;
-  sqlite3_stmt *pSeek;            /* SELECT to read row iDel from %_data */
+  sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
  int rc;                         /* Return code */
+  int rc2;                        /* sqlite3_reset() return code */
+  int iCol;
+  Fts5InsertCtx ctx;

-  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
-  if( rc==SQLITE_OK ){
-    int rc2;
+  if( apVal==0 ){
+    rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
+    if( rc!=SQLITE_OK ) return rc;
    sqlite3_bind_int64(pSeek, 1, iDel);
-    if( sqlite3_step(pSeek)==SQLITE_ROW ){
-      int iCol;
-      Fts5InsertCtx ctx;
-      ctx.pStorage = p;
-      ctx.iCol = -1;
-      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
-      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
-        if( pConfig->abUnindexed[iCol-1] ) continue;
-        ctx.szCol = 0;
-        rc = sqlite3Fts5Tokenize(pConfig, 
-            FTS5_TOKENIZE_DOCUMENT,
-            (const char*)sqlite3_column_text(pSeek, iCol),
-            sqlite3_column_bytes(pSeek, iCol),
-            (void*)&ctx,
-            fts5StorageInsertCallback
-        );
-        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
-      }
-      p->nTotalRow--;
+    if( sqlite3_step(pSeek)!=SQLITE_ROW ){
+      return sqlite3_reset(pSeek);
    }
-    rc2 = sqlite3_reset(pSeek);
-    if( rc==SQLITE_OK ) rc = rc2;
  }

+  ctx.pStorage = p;
+  ctx.iCol = -1;
+  rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
+  for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
+    if( pConfig->abUnindexed[iCol-1]==0 ){
+      const char *zText;
+      int nText;
+      if( pSeek ){
+        zText = (const char*)sqlite3_column_text(pSeek, iCol);
+        nText = sqlite3_column_bytes(pSeek, iCol);
+      }else{
+        zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
+        nText = sqlite3_value_bytes(apVal[iCol-1]);
+      }
+      ctx.szCol = 0;
+      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
+          zText, nText, (void*)&ctx, fts5StorageInsertCallback
+      );
+      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
+    }
+  }
+  p->nTotalRow--;
+
+  rc2 = sqlite3_reset(pSeek);
+  if( rc==SQLITE_OK ) rc = rc2;
  return rc;
 }

@@ -490,16 +503,17 @@ static int fts5StorageSaveTotals(Fts5Storage *p){
 /*
 ** Remove a row from the FTS table.
 */
-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
  Fts5Config *pConfig = p->pConfig;
  int rc;
  sqlite3_stmt *pDel = 0;

+  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 );
  rc = fts5StorageLoadTotals(p, 1);

  /* Delete the index records */
  if( rc==SQLITE_OK ){
-    rc = fts5StorageDeleteFromIndex(p, iDel);
+    rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
  }

  /* Delete the %_docsize record */
@@ -513,62 +527,9 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
  }

  /* Delete the %_content record */
-  if( rc==SQLITE_OK ){
-    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
-  }
-  if( rc==SQLITE_OK ){
-    sqlite3_bind_int64(pDel, 1, iDel);
-    sqlite3_step(pDel);
-    rc = sqlite3_reset(pDel);
-  }
-
-  /* Write the averages record */
-  if( rc==SQLITE_OK ){
-    rc = fts5StorageSaveTotals(p);
-  }
-
-  return rc;
-}
-
-int sqlite3Fts5StorageSpecialDelete(
-  Fts5Storage *p, 
-  i64 iDel, 
-  sqlite3_value **apVal
-){
-  Fts5Config *pConfig = p->pConfig;
-  int rc;
-  sqlite3_stmt *pDel = 0;
-
-  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL );
-  rc = fts5StorageLoadTotals(p, 1);
-
-  /* Delete the index records */
-  if( rc==SQLITE_OK ){
-    int iCol;
-    Fts5InsertCtx ctx;
-    ctx.pStorage = p;
-    ctx.iCol = -1;
-
-    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
-    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
-      if( pConfig->abUnindexed[iCol] ) continue;
-      ctx.szCol = 0;
-      rc = sqlite3Fts5Tokenize(pConfig, 
-        FTS5_TOKENIZE_DOCUMENT,
-        (const char*)sqlite3_value_text(apVal[iCol]),
-        sqlite3_value_bytes(apVal[iCol]),
-        (void*)&ctx,
-        fts5StorageInsertCallback
-      );
-      p->aTotalSize[iCol] -= (i64)ctx.szCol;
-    }
-    p->nTotalRow--;
-  }
-
-  /* Delete the %_docsize record */
-  if( pConfig->bColumnsize ){
+  if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
    if( rc==SQLITE_OK ){
-      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
+      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
    }
    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pDel, 1, iDel);
@@ -725,17 +686,7 @@ int sqlite3Fts5StorageContentInsert(
  }else{
    sqlite3_stmt *pInsert = 0;    /* Statement to write %_content table */
    int i;                        /* Counter variable */
-#if 0
-    if( eConflict==SQLITE_REPLACE ){
-      eStmt = FTS5_STMT_REPLACE_CONTENT;
-      rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
-    }else{
-      eStmt = FTS5_STMT_INSERT_CONTENT;
-    }
-#endif
-    if( rc==SQLITE_OK ){
-      rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
-    }
+    rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
    for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
      rc = sqlite3_bind_value(pInsert, i, apVal[i]);
    }
@@ -833,14 +784,16 @@ struct Fts5IntegrityCtx {
  int iCol;
  int szCol;
  u64 cksum;
+  Fts5Termset *pTermset;
  Fts5Config *pConfig;
 };

+
 /*
 ** Tokenization callback used by integrity check.
 */
 static int fts5StorageIntegrityCallback(
-  void *pContext,                 /* Pointer to Fts5InsertCtx object */
+  void *pContext,                 /* Pointer to Fts5IntegrityCtx object */
  int tflags,
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
@@ -848,13 +801,56 @@ static int fts5StorageIntegrityCallback(
  int iEnd                        /* End offset of token */
 ){
  Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
+  Fts5Termset *pTermset = pCtx->pTermset;
+  int bPresent;
+  int ii;
+  int rc = SQLITE_OK;
+  int iPos;
+  int iCol;
+
  if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
    pCtx->szCol++;
  }
-  pCtx->cksum ^= sqlite3Fts5IndexCksum(
-      pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken
-  );
-  return SQLITE_OK;
+
+  switch( pCtx->pConfig->eDetail ){
+    case FTS5_DETAIL_FULL:
+      iPos = pCtx->szCol-1;
+      iCol = pCtx->iCol;
+      break;
+
+    case FTS5_DETAIL_COLUMNS:
+      iPos = pCtx->iCol;
+      iCol = 0;
+      break;
+
+    default:
+      assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE );
+      iPos = 0;
+      iCol = 0;
+      break;
+  }
+
+  rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
+  if( rc==SQLITE_OK && bPresent==0 ){
+    pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
+        pCtx->iRowid, iCol, iPos, 0, pToken, nToken
+    );
+  }
+
+  for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){
+    const int nChar = pCtx->pConfig->aPrefix[ii];
+    int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
+    if( nByte ){
+      rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent);
+      if( bPresent==0 ){
+        pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
+            pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte
+        );
+      }
+    }
+  }
+
+  return rc;
 }

 /*
@@ -890,22 +886,37 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
      if( pConfig->bColumnsize ){
        rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
      }
+      if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){
+        rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
+      }
      for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
        if( pConfig->abUnindexed[i] ) continue;
        ctx.iCol = i;
        ctx.szCol = 0;
-        rc = sqlite3Fts5Tokenize(pConfig, 
-            FTS5_TOKENIZE_DOCUMENT,
-            (const char*)sqlite3_column_text(pScan, i+1),
-            sqlite3_column_bytes(pScan, i+1),
-            (void*)&ctx,
-            fts5StorageIntegrityCallback
-        );
-        if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
+        if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+          rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
+        }
+        if( rc==SQLITE_OK ){
+          rc = sqlite3Fts5Tokenize(pConfig, 
+              FTS5_TOKENIZE_DOCUMENT,
+              (const char*)sqlite3_column_text(pScan, i+1),
+              sqlite3_column_bytes(pScan, i+1),
+              (void*)&ctx,
+              fts5StorageIntegrityCallback
+          );
+        }
+        if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
          rc = FTS5_CORRUPT;
        }
        aTotalSize[i] += ctx.szCol;
+        if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+          sqlite3Fts5TermsetFree(ctx.pTermset);
+          ctx.pTermset = 0;
+        }
      }
+      sqlite3Fts5TermsetFree(ctx.pTermset);
+      ctx.pTermset = 0;
+
      if( rc!=SQLITE_OK ) break;
    }
    rc2 = sqlite3_reset(pScan);
@@ -924,12 +935,12 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
  /* Check that the %_docsize and %_content tables contain the expected
  ** number of rows.  */
  if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
-    i64 nRow;
+    i64 nRow = 0;
    rc = fts5StorageCount(p, "content", &nRow);
    if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
  }
  if( rc==SQLITE_OK && pConfig->bColumnsize ){
-    i64 nRow;
+    i64 nRow = 0;
    rc = fts5StorageCount(p, "docsize", &nRow);
    if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
  }
@@ -23,7 +23,8 @@
 #include <assert.h>

 extern int sqlite3_fts5_may_be_corrupt;
-extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *);
+extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*);
+extern int sqlite3Fts5TestRegisterTok(sqlite3*, fts5_api*);

 /*************************************************************************
 ** This is a copy of the first part of the SqliteDb structure in 
@@ -235,6 +236,8 @@ static int xF5tApi(
    { "xGetAuxdata",       1, "CLEAR" },              /* 13 */
    { "xSetAuxdataInt",    1, "INTEGER" },            /* 14 */
    { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
+    { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
+    { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
    { 0, 0, 0}
  };

@@ -431,6 +434,66 @@ static int xF5tApi(
      break;
    }

+    CASE(16, "xPhraseForeach") {
+      int iPhrase;
+      int iCol;
+      int iOff;
+      const char *zColvar;
+      const char *zOffvar;
+      Tcl_Obj *pScript = objv[5];
+      Fts5PhraseIter iter;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
+      zColvar = Tcl_GetString(objv[3]);
+      zOffvar = Tcl_GetString(objv[4]);
+
+      rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
+      if( rc!=SQLITE_OK ){
+        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        return TCL_ERROR;
+      }
+      for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
+        Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
+        Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0);
+        rc = Tcl_EvalObjEx(interp, pScript, 0);
+        if( rc==TCL_CONTINUE ) rc = TCL_OK;
+        if( rc!=TCL_OK ){
+          if( rc==TCL_BREAK ) rc = TCL_OK;
+          break;
+        }
+      }
+
+      break;
+    }
+
+    CASE(17, "xPhraseColumnForeach") {
+      int iPhrase;
+      int iCol;
+      const char *zColvar;
+      Tcl_Obj *pScript = objv[4];
+      Fts5PhraseIter iter;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
+      zColvar = Tcl_GetString(objv[3]);
+
+      rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
+      if( rc!=SQLITE_OK ){
+        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        return TCL_ERROR;
+      }
+      for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
+        Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
+        rc = Tcl_EvalObjEx(interp, pScript, 0);
+        if( rc==TCL_CONTINUE ) rc = TCL_OK;
+        if( rc!=TCL_OK ){
+          if( rc==TCL_BREAK ) rc = TCL_OK;
+          break;
+        }
+      }
+
+      break;
+    }
+
    default: 
      assert( 0 );
      break;
@@ -1020,6 +1083,32 @@ static int f5tRegisterMatchinfo(
  return TCL_OK;
 }

+static int f5tRegisterTok(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  sqlite3 *db = 0;
+  fts5_api *pApi = 0;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
+    return TCL_ERROR;
+  }
+
+  rc = sqlite3Fts5TestRegisterTok(db, pApi);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+
 /*
 ** Entry point.
 */
@@ -1035,7 +1124,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
    { "sqlite3_fts5_create_function",    f5tCreateFunction, 0 },
    { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
    { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
-    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }
+    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
+    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }
  };
  int i;
  F5tTokenizerContext *pContext;
@@ -134,7 +134,7 @@ static int fts5MatchinfoXCb(
  int iPrev = -1;

  for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); 
-      iOff>=0; 
+      iCol>=0; 
      pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
  ){
    aOut[iCol*3+1]++;
@@ -211,18 +211,31 @@ static int fts5MatchinfoLocalCb(
  int rc = SQLITE_OK;

  switch( f ){
-    case 'b': 
+    case 'b': {
+      int iPhrase;
+      int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
+      for(i=0; i<nInt; i++) aOut[i] = 0;
+
+      for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
+        Fts5PhraseIter iter;
+        int iCol;
+        for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
+            iCol>=0; 
+            pApi->xPhraseNextColumn(pFts, &iter, &iCol)
+        ){
+          aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
+        }
+      }
+
+      break;
+    }
+
    case 'x':
    case 'y': {
      int nMul = (f=='x' ? 3 : 1);
      int iPhrase;

-      if( f=='b' ){
-        int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
-        for(i=0; i<nInt; i++) aOut[i] = 0;
-      }else{
-        for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
-      }
+      for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;

      for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
        Fts5PhraseIter iter;
@@ -0,0 +1,482 @@
+/*
+** 2013 Apr 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains code for the "fts5tokenize" virtual table module.
+** An fts5tokenize virtual table is created as follows:
+**
+**   CREATE VIRTUAL TABLE <tbl> USING fts5tokenize(
+**       <tokenizer-name>, <arg-1>, ...
+**   );
+**
+** The table created has the following schema:
+**
+**   CREATE TABLE <tbl>(input HIDDEN, token, start, end, position)
+**
+** When queried, the query must include a WHERE clause of type:
+**
+**   input = <string>
+**
+** The virtual table module tokenizes this <string>, using the FTS3 
+** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE 
+** statement and returns one row for each token in the result. With
+** fields set as follows:
+**
+**   input:   Always set to a copy of <string>
+**   token:   A token from the input.
+**   start:   Byte offset of the token within the input <string>.
+**   end:     Byte offset of the byte immediately following the end of the
+**            token within the input string.
+**   pos:     Token offset of token within input.
+**
+*/
+#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)
+
+#include <fts5.h>
+#include <string.h>
+#include <assert.h>
+
+typedef struct Fts5tokTable Fts5tokTable;
+typedef struct Fts5tokCursor Fts5tokCursor;
+typedef struct Fts5tokRow Fts5tokRow;
+
+/*
+** Virtual table structure.
+*/
+struct Fts5tokTable {
+  sqlite3_vtab base;              /* Base class used by SQLite core */
+  fts5_tokenizer tok;             /* Tokenizer functions */
+  Fts5Tokenizer *pTok;            /* Tokenizer instance */
+};
+
+/*
+** A container for a rows values.
+*/
+struct Fts5tokRow {
+  char *zToken;
+  int iStart;
+  int iEnd;
+  int iPos;
+};
+
+/*
+** Virtual table cursor structure.
+*/
+struct Fts5tokCursor {
+  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
+  int iRowid;                     /* Current 'rowid' value */
+  char *zInput;                   /* Input string */
+  int nRow;                       /* Number of entries in aRow[] */
+  Fts5tokRow *aRow;               /* Array of rows to return */
+};
+
+static void fts5tokDequote(char *z){
+  char q = z[0];
+
+  if( q=='[' || q=='\'' || q=='"' || q=='`' ){
+    int iIn = 1;
+    int iOut = 0;
+    if( q=='[' ) q = ']';  
+
+    while( z[iIn] ){
+      if( z[iIn]==q ){
+        if( z[iIn+1]!=q ){
+          /* Character iIn was the close quote. */
+          iIn++;
+          break;
+        }else{
+          /* Character iIn and iIn+1 form an escaped quote character. Skip
+          ** the input cursor past both and copy a single quote character 
+          ** to the output buffer. */
+          iIn += 2;
+          z[iOut++] = q;
+        }
+      }else{
+        z[iOut++] = z[iIn++];
+      }
+    }
+
+    z[iOut] = '\0';
+  }
+}
+
+/*
+** The second argument, argv[], is an array of pointers to nul-terminated
+** strings. This function makes a copy of the array and strings into a 
+** single block of memory. It then dequotes any of the strings that appear
+** to be quoted.
+**
+** If successful, output parameter *pazDequote is set to point at the
+** array of dequoted strings and SQLITE_OK is returned. The caller is
+** responsible for eventually calling sqlite3_free() to free the array
+** in this case. Or, if an error occurs, an SQLite error code is returned.
+** The final value of *pazDequote is undefined in this case.
+*/
+static int fts5tokDequoteArray(
+  int argc,                       /* Number of elements in argv[] */
+  const char * const *argv,       /* Input array */
+  char ***pazDequote              /* Output array */
+){
+  int rc = SQLITE_OK;             /* Return code */
+  if( argc==0 ){
+    *pazDequote = 0;
+  }else{
+    int i;
+    int nByte = 0;
+    char **azDequote;
+
+    for(i=0; i<argc; i++){
+      nByte += (int)(strlen(argv[i]) + 1);
+    }
+
+    *pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
+    if( azDequote==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      char *pSpace = (char *)&azDequote[argc];
+      for(i=0; i<argc; i++){
+        int n = (int)strlen(argv[i]);
+        azDequote[i] = pSpace;
+        memcpy(pSpace, argv[i], n+1);
+        fts5tokDequote(pSpace);
+        pSpace += (n+1);
+      }
+    }
+  }
+
+  return rc;
+}
+
+/*
+** Schema of the tokenizer table.
+*/
+#define FTS3_TOK_SCHEMA "CREATE TABLE x(input HIDDEN, token, start, end, position)"
+
+/*
+** This function does all the work for both the xConnect and xCreate methods.
+** These tables have no persistent representation of their own, so xConnect
+** and xCreate are identical operations.
+**
+**   argv[0]: module name
+**   argv[1]: database name 
+**   argv[2]: table name
+**   argv[3]: first argument (tokenizer name)
+*/
+static int fts5tokConnectMethod(
+  sqlite3 *db,                    /* Database connection */
+  void *pCtx,                     /* Pointer to fts5_api object */
+  int argc,                       /* Number of elements in argv array */
+  const char * const *argv,       /* xCreate/xConnect argument array */
+  sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
+  char **pzErr                    /* OUT: sqlite3_malloc'd error message */
+){
+  fts5_api *pApi = (fts5_api*)pCtx;
+  Fts5tokTable *pTab = 0;
+  int rc;
+  char **azDequote = 0;
+  int nDequote;
+
+  rc = sqlite3_declare_vtab(db, 
+       "CREATE TABLE x(input HIDDEN, token, start, end, position)"
+  );
+
+  if( rc==SQLITE_OK ){
+    nDequote = argc-3;
+    rc = fts5tokDequoteArray(nDequote, &argv[3], &azDequote);
+  }
+
+  if( rc==SQLITE_OK ){
+    pTab = (Fts5tokTable*)sqlite3_malloc(sizeof(Fts5tokTable));
+    if( pTab==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memset(pTab, 0, sizeof(Fts5tokTable));
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    void *pTokCtx = 0;
+    const char *zModule = 0;
+    if( nDequote>0 ){
+      zModule = azDequote[0];
+    }
+
+    rc = pApi->xFindTokenizer(pApi, zModule, &pTokCtx, &pTab->tok);
+    if( rc==SQLITE_OK ){
+      const char **azArg = (const char **)&azDequote[1];
+      int nArg = nDequote>0 ? nDequote-1 : 0;
+      rc = pTab->tok.xCreate(pTokCtx, azArg, nArg, &pTab->pTok);
+    }
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(pTab);
+    pTab = 0;
+  }
+
+  *ppVtab = (sqlite3_vtab*)pTab;
+  sqlite3_free(azDequote);
+  return rc;
+}
+
+/*
+** This function does the work for both the xDisconnect and xDestroy methods.
+** These tables have no persistent representation of their own, so xDisconnect
+** and xDestroy are identical operations.
+*/
+static int fts5tokDisconnectMethod(sqlite3_vtab *pVtab){
+  Fts5tokTable *pTab = (Fts5tokTable *)pVtab;
+  if( pTab->pTok ){
+    pTab->tok.xDelete(pTab->pTok);
+  }
+  sqlite3_free(pTab);
+  return SQLITE_OK;
+}
+
+/*
+** xBestIndex - Analyze a WHERE and ORDER BY clause.
+*/
+static int fts5tokBestIndexMethod(
+  sqlite3_vtab *pVTab, 
+  sqlite3_index_info *pInfo
+){
+  int i;
+
+  for(i=0; i<pInfo->nConstraint; i++){
+    if( pInfo->aConstraint[i].usable 
+     && pInfo->aConstraint[i].iColumn==0 
+     && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ 
+    ){
+      pInfo->idxNum = 1;
+      pInfo->aConstraintUsage[i].argvIndex = 1;
+      pInfo->aConstraintUsage[i].omit = 1;
+      pInfo->estimatedCost = 1;
+      return SQLITE_OK;
+    }
+  }
+
+  pInfo->idxNum = 0;
+  assert( pInfo->estimatedCost>1000000.0 );
+
+  return SQLITE_OK;
+}
+
+/*
+** xOpen - Open a cursor.
+*/
+static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+  Fts5tokCursor *pCsr;
+
+  pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor));
+  if( pCsr==0 ){
+    return SQLITE_NOMEM;
+  }
+  memset(pCsr, 0, sizeof(Fts5tokCursor));
+
+  *ppCsr = (sqlite3_vtab_cursor *)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Reset the tokenizer cursor passed as the only argument. As if it had
+** just been returned by fts5tokOpenMethod().
+*/
+static void fts5tokResetCursor(Fts5tokCursor *pCsr){
+  int i;
+  for(i=0; i<pCsr->nRow; i++){
+    sqlite3_free(pCsr->aRow[i].zToken);
+  }
+  sqlite3_free(pCsr->zInput);
+  sqlite3_free(pCsr->aRow);
+  pCsr->zInput = 0;
+  pCsr->aRow = 0;
+  pCsr->nRow = 0;
+  pCsr->iRowid = 0;
+}
+
+/*
+** xClose - Close a cursor.
+*/
+static int fts5tokCloseMethod(sqlite3_vtab_cursor *pCursor){
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  fts5tokResetCursor(pCsr);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** xNext - Advance the cursor to the next row, if any.
+*/
+static int fts5tokNextMethod(sqlite3_vtab_cursor *pCursor){
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  pCsr->iRowid++;
+  return SQLITE_OK;
+}
+
+static int fts5tokCb(
+  void *pCtx,         /* Pointer to Fts5tokCursor */
+  int tflags,         /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken, /* Pointer to buffer containing token */
+  int nToken,         /* Size of token in bytes */
+  int iStart,         /* Byte offset of token within input text */
+  int iEnd            /* Byte offset of end of token within input text */
+){
+  Fts5tokCursor *pCsr = (Fts5tokCursor*)pCtx;
+  Fts5tokRow *pRow;
+
+  if( (pCsr->nRow & (pCsr->nRow-1))==0 ){
+    int nNew = pCsr->nRow ? pCsr->nRow*2 : 32;
+    Fts5tokRow *aNew;
+    aNew = (Fts5tokRow*)sqlite3_realloc(pCsr->aRow, nNew*sizeof(Fts5tokRow));
+    if( aNew==0 ) return SQLITE_NOMEM;
+    memset(&aNew[pCsr->nRow], 0, sizeof(Fts5tokRow)*(nNew-pCsr->nRow));
+    pCsr->aRow = aNew;
+  }
+
+  pRow = &pCsr->aRow[pCsr->nRow];
+  pRow->iStart = iStart;
+  pRow->iEnd = iEnd;
+  if( pCsr->nRow ){
+    pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1);
+  }
+  pRow->zToken = sqlite3_malloc(nToken+1);
+  if( pRow->zToken==0 ) return SQLITE_NOMEM;
+  memcpy(pRow->zToken, pToken, nToken);
+  pRow->zToken[nToken] = 0;
+  pCsr->nRow++;
+
+  return SQLITE_OK;
+}
+
+/*
+** xFilter - Initialize a cursor to point at the start of its data.
+*/
+static int fts5tokFilterMethod(
+  sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
+  int idxNum,                     /* Strategy index */
+  const char *idxStr,             /* Unused */
+  int nVal,                       /* Number of elements in apVal */
+  sqlite3_value **apVal           /* Arguments for the indexing scheme */
+){
+  int rc = SQLITE_ERROR;
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  Fts5tokTable *pTab = (Fts5tokTable *)(pCursor->pVtab);
+
+  fts5tokResetCursor(pCsr);
+  if( idxNum==1 ){
+    const char *zByte = (const char *)sqlite3_value_text(apVal[0]);
+    int nByte = sqlite3_value_bytes(apVal[0]);
+    pCsr->zInput = sqlite3_malloc(nByte+1);
+    if( pCsr->zInput==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memcpy(pCsr->zInput, zByte, nByte);
+      pCsr->zInput[nByte] = 0;
+      rc = pTab->tok.xTokenize(
+          pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb
+      );
+    }
+  }
+
+  if( rc!=SQLITE_OK ) return rc;
+  return fts5tokNextMethod(pCursor);
+}
+
+/*
+** xEof - Return true if the cursor is at EOF, or false otherwise.
+*/
+static int fts5tokEofMethod(sqlite3_vtab_cursor *pCursor){
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  return (pCsr->iRowid>pCsr->nRow);
+}
+
+/*
+** xColumn - Return a column value.
+*/
+static int fts5tokColumnMethod(
+  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
+  sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
+  int iCol                        /* Index of column to read value from */
+){
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  Fts5tokRow *pRow = &pCsr->aRow[pCsr->iRowid-1];
+
+  /* CREATE TABLE x(input, token, start, end, position) */
+  switch( iCol ){
+    case 0:
+      sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT);
+      break;
+    case 1:
+      sqlite3_result_text(pCtx, pRow->zToken, -1, SQLITE_TRANSIENT);
+      break;
+    case 2:
+      sqlite3_result_int(pCtx, pRow->iStart);
+      break;
+    case 3:
+      sqlite3_result_int(pCtx, pRow->iEnd);
+      break;
+    default:
+      assert( iCol==4 );
+      sqlite3_result_int(pCtx, pRow->iPos);
+      break;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** xRowid - Return the current rowid for the cursor.
+*/
+static int fts5tokRowidMethod(
+  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
+  sqlite_int64 *pRowid            /* OUT: Rowid value */
+){
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  *pRowid = (sqlite3_int64)pCsr->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Register the fts5tok module with database connection db. Return SQLITE_OK
+** if successful or an error code if sqlite3_create_module() fails.
+*/
+int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
+  static const sqlite3_module fts5tok_module = {
+     0,                           /* iVersion      */
+     fts5tokConnectMethod,        /* xCreate       */
+     fts5tokConnectMethod,        /* xConnect      */
+     fts5tokBestIndexMethod,      /* xBestIndex    */
+     fts5tokDisconnectMethod,     /* xDisconnect   */
+     fts5tokDisconnectMethod,     /* xDestroy      */
+     fts5tokOpenMethod,           /* xOpen         */
+     fts5tokCloseMethod,          /* xClose        */
+     fts5tokFilterMethod,         /* xFilter       */
+     fts5tokNextMethod,           /* xNext         */
+     fts5tokEofMethod,            /* xEof          */
+     fts5tokColumnMethod,         /* xColumn       */
+     fts5tokRowidMethod,          /* xRowid        */
+     0,                           /* xUpdate       */
+     0,                           /* xBegin        */
+     0,                           /* xSync         */
+     0,                           /* xCommit       */
+     0,                           /* xRollback     */
+     0,                           /* xFindFunction */
+     0,                           /* xRename       */
+     0,                           /* xSavepoint    */
+     0,                           /* xRelease      */
+     0                            /* xRollbackTo   */
+  };
+  int rc;                         /* Return code */
+
+  rc = sqlite3_create_module(db, "fts5tokenize", &fts5tok_module, (void*)pApi);
+  return rc;
+}
+
+#endif /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) */
@@ -242,7 +242,7 @@ static int fts5UnicodeAddExceptions(
  int bTokenChars                 /* 1 for 'tokenchars', 0 for 'separators' */
 ){
  int rc = SQLITE_OK;
-  int n = strlen(z);
+  int n = (int)strlen(z);
  int *aNew;

  if( n>0 ){
@@ -256,7 +256,7 @@ static int fts5UnicodeAddExceptions(
        int bToken;
        READ_UTF8(zCsr, zTerm, iCode);
        if( iCode<128 ){
-          p->aTokenChar[iCode] = bTokenChars;
+          p->aTokenChar[iCode] = (unsigned char)bTokenChars;
        }else{
          bToken = sqlite3Fts5UnicodeIsalnum(iCode);
          assert( (bToken==0 || bToken==1) ); 
@@ -1220,7 +1220,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */

-  for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
+  for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
    rc = pApi->xCreateTokenizer(pApi,
        aBuiltin[i].zName,
        (void*)pApi,
@@ -333,7 +333,10 @@ int sqlite3Fts5PutVarint(unsigned char *p, u64 v){


 int sqlite3Fts5GetVarintLen(u32 iVal){
+#if 0
  if( iVal<(1 << 7 ) ) return 1;
+#endif
+  assert( iVal>=(1 << 7) );
  if( iVal<(1 << 14) ) return 2;
  if( iVal<(1 << 21) ) return 3;
  if( iVal<(1 << 28) ) return 4;
@@ -55,16 +55,18 @@ struct Fts5VocabCursor {
  int bEof;                       /* True if this cursor is at EOF */
  Fts5IndexIter *pIter;           /* Term/rowid iterator object */

+  int nLeTerm;                    /* Size of zLeTerm in bytes */
+  char *zLeTerm;                  /* (term <= $zLeTerm) paramater, or NULL */
+
  /* These are used by 'col' tables only */
-  int nCol;
+  Fts5Config *pConfig;            /* Fts5 table configuration */
  int iCol;
  i64 *aCnt;
  i64 *aDoc;

-  /* Output values */
+  /* Output values used by 'row' and 'col' tables */
  i64 rowid;                      /* This table's current rowid value */
  Fts5Buffer term;                /* Current value of 'term' column */
-  i64 aVal[3];                    /* Up to three columns left of 'term' */
 };

 #define FTS5_VOCAB_COL    0
@@ -73,6 +75,14 @@ struct Fts5VocabCursor {
 #define FTS5_VOCAB_COL_SCHEMA  "term, col, doc, cnt"
 #define FTS5_VOCAB_ROW_SCHEMA  "term, doc, cnt"

+/*
+** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
+*/
+#define FTS5_VOCAB_TERM_EQ 0x01
+#define FTS5_VOCAB_TERM_GE 0x02
+#define FTS5_VOCAB_TERM_LE 0x04
+
+
 /*
 ** Translate a string containing an fts5vocab table type to an 
 ** FTS5_VOCAB_XXX constant. If successful, set *peType to the output
@@ -168,13 +178,13 @@ static int fts5VocabInitVtab(
    const char *zDb = bDb ? argv[3] : argv[1];
    const char *zTab = bDb ? argv[4] : argv[3];
    const char *zType = bDb ? argv[5] : argv[4];
-    int nDb = strlen(zDb)+1; 
-    int nTab = strlen(zTab)+1;
-    int eType;
+    int nDb = (int)strlen(zDb)+1; 
+    int nTab = (int)strlen(zTab)+1;
+    int eType = 0;
    
    rc = fts5VocabTableType(zType, pzErr, &eType);
    if( rc==SQLITE_OK ){
-      assert( eType>=0 && eType<sizeof(azSchema)/sizeof(azSchema[0]) );
+      assert( eType>=0 && eType<ArraySize(azSchema) );
      rc = sqlite3_declare_vtab(db, azSchema[eType]);
    }

@@ -230,6 +240,45 @@ static int fts5VocabBestIndexMethod(
  sqlite3_vtab *pVTab, 
  sqlite3_index_info *pInfo
 ){
+  int i;
+  int iTermEq = -1;
+  int iTermGe = -1;
+  int iTermLe = -1;
+  int idxNum = 0;
+  int nArg = 0;
+
+  for(i=0; i<pInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
+    if( p->usable==0 ) continue;
+    if( p->iColumn==0 ){          /* term column */
+      if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i;
+      if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i;
+      if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i;
+      if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i;
+      if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i;
+    }
+  }
+
+  if( iTermEq>=0 ){
+    idxNum |= FTS5_VOCAB_TERM_EQ;
+    pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg;
+    pInfo->estimatedCost = 100;
+  }else{
+    pInfo->estimatedCost = 1000000;
+    if( iTermGe>=0 ){
+      idxNum |= FTS5_VOCAB_TERM_GE;
+      pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg;
+      pInfo->estimatedCost = pInfo->estimatedCost / 2;
+    }
+    if( iTermLe>=0 ){
+      idxNum |= FTS5_VOCAB_TERM_LE;
+      pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg;
+      pInfo->estimatedCost = pInfo->estimatedCost / 2;
+    }
+  }
+
+  pInfo->idxNum = idxNum;
+
  return SQLITE_OK;
 }

@@ -242,12 +291,11 @@ static int fts5VocabOpenMethod(
 ){
  Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
  Fts5Index *pIndex = 0;
-  int nCol = 0;
+  Fts5Config *pConfig = 0;
  Fts5VocabCursor *pCsr = 0;
  int rc = SQLITE_OK;
  sqlite3_stmt *pStmt = 0;
  char *zSql = 0;
-  int nByte;

  zSql = sqlite3Fts5Mprintf(&rc,
      "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
@@ -262,7 +310,7 @@ static int fts5VocabOpenMethod(

  if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
    i64 iId = sqlite3_column_int64(pStmt, 0);
-    pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &nCol);
+    pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig);
  }

  if( rc==SQLITE_OK && pIndex==0 ){
@@ -276,14 +324,17 @@ static int fts5VocabOpenMethod(
    }
  }

-  nByte = nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
-  pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
+  if( rc==SQLITE_OK ){
+    int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
+    pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
+  }
+
  if( pCsr ){
    pCsr->pIndex = pIndex;
    pCsr->pStmt = pStmt;
-    pCsr->nCol = nCol;
+    pCsr->pConfig = pConfig;
    pCsr->aCnt = (i64*)&pCsr[1];
-    pCsr->aDoc = &pCsr->aCnt[nCol];
+    pCsr->aDoc = &pCsr->aCnt[pConfig->nCol];
  }else{
    sqlite3_finalize(pStmt);
  }
@@ -296,6 +347,9 @@ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
  pCsr->rowid = 0;
  sqlite3Fts5IterClose(pCsr->pIter);
  pCsr->pIter = 0;
+  sqlite3_free(pCsr->zLeTerm);
+  pCsr->nLeTerm = -1;
+  pCsr->zLeTerm = 0;
 }

 /*
@@ -319,16 +373,17 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
  Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
  int rc = SQLITE_OK;
+  int nCol = pCsr->pConfig->nCol;

  pCsr->rowid++;

  if( pTab->eType==FTS5_VOCAB_COL ){
-    for(pCsr->iCol++; pCsr->iCol<pCsr->nCol; pCsr->iCol++){
-      if( pCsr->aCnt[pCsr->iCol] ) break;
+    for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
+      if( pCsr->aDoc[pCsr->iCol] ) break;
    }
  }

-  if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=pCsr->nCol ){
+  if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){
    if( sqlite3Fts5IterEof(pCsr->pIter) ){
      pCsr->bEof = 1;
    }else{
@@ -336,53 +391,93 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
      int nTerm;

      zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
+      if( pCsr->nLeTerm>=0 ){
+        int nCmp = MIN(nTerm, pCsr->nLeTerm);
+        int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
+        if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
+          pCsr->bEof = 1;
+          return SQLITE_OK;
+        }
+      }
+
      sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
-      memset(pCsr->aVal, 0, sizeof(pCsr->aVal));
-      memset(pCsr->aCnt, 0, pCsr->nCol * sizeof(i64));
-      memset(pCsr->aDoc, 0, pCsr->nCol * sizeof(i64));
+      memset(pCsr->aCnt, 0, nCol * sizeof(i64));
+      memset(pCsr->aDoc, 0, nCol * sizeof(i64));
      pCsr->iCol = 0;

      assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
      while( rc==SQLITE_OK ){
-        i64 dummy;
        const u8 *pPos; int nPos;   /* Position list */
        i64 iPos = 0;               /* 64-bit position read from poslist */
        int iOff = 0;               /* Current offset within position list */

-        rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy);
-        if( rc==SQLITE_OK ){
-          if( pTab->eType==FTS5_VOCAB_ROW ){
-            while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
-              pCsr->aVal[1]++;
-            }
-            pCsr->aVal[0]++;
-          }else{
-            int iCol = -1;
-            while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
-              int ii = FTS5_POS2COLUMN(iPos);
-              pCsr->aCnt[ii]++;
-              if( iCol!=ii ){
-                pCsr->aDoc[ii]++;
-                iCol = ii;
+        pPos = pCsr->pIter->pData;
+        nPos = pCsr->pIter->nData;
+        switch( pCsr->pConfig->eDetail ){
+          case FTS5_DETAIL_FULL:
+            pPos = pCsr->pIter->pData;
+            nPos = pCsr->pIter->nData;
+            if( pTab->eType==FTS5_VOCAB_ROW ){
+              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+                pCsr->aCnt[0]++;
+              }
+              pCsr->aDoc[0]++;
+            }else{
+              int iCol = -1;
+              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+                int ii = FTS5_POS2COLUMN(iPos);
+                pCsr->aCnt[ii]++;
+                if( iCol!=ii ){
+                  if( ii>=nCol ){
+                    rc = FTS5_CORRUPT;
+                    break;
+                  }
+                  pCsr->aDoc[ii]++;
+                  iCol = ii;
+                }
              }
            }
-          }
+            break;
+
+          case FTS5_DETAIL_COLUMNS:
+            if( pTab->eType==FTS5_VOCAB_ROW ){
+              pCsr->aDoc[0]++;
+            }else{
+              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
+                assert_nc( iPos>=0 && iPos<nCol );
+                if( iPos>=nCol ){
+                  rc = FTS5_CORRUPT;
+                  break;
+                }
+                pCsr->aDoc[iPos]++;
+              }
+            }
+            break;
+
+          default: 
+            assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE );
+            pCsr->aDoc[0]++;
+            break;
+        }
+
+        if( rc==SQLITE_OK ){
          rc = sqlite3Fts5IterNextScan(pCsr->pIter);
        }
+
        if( rc==SQLITE_OK ){
          zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
-          if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break;
+          if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){
+            break;
+          }
          if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
        }
      }
    }
  }

-  if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
-    while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++;
-    pCsr->aVal[0] = pCsr->iCol;
-    pCsr->aVal[1] = pCsr->aDoc[pCsr->iCol];
-    pCsr->aVal[2] = pCsr->aCnt[pCsr->iCol];
+  if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
+    while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
+    assert( pCsr->iCol<pCsr->pConfig->nCol );
  }
  return rc;
 }
@@ -398,11 +493,47 @@ static int fts5VocabFilterMethod(
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
 ){
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
-  int rc;
-  const int flags = FTS5INDEX_QUERY_SCAN;
+  int rc = SQLITE_OK;
+
+  int iVal = 0;
+  int f = FTS5INDEX_QUERY_SCAN;
+  const char *zTerm = 0;
+  int nTerm = 0;
+
+  sqlite3_value *pEq = 0;
+  sqlite3_value *pGe = 0;
+  sqlite3_value *pLe = 0;

  fts5VocabResetCursor(pCsr);
-  rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, 0, &pCsr->pIter);
+  if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
+  if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
+  if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
+
+  if( pEq ){
+    zTerm = (const char *)sqlite3_value_text(pEq);
+    nTerm = sqlite3_value_bytes(pEq);
+    f = 0;
+  }else{
+    if( pGe ){
+      zTerm = (const char *)sqlite3_value_text(pGe);
+      nTerm = sqlite3_value_bytes(pGe);
+    }
+    if( pLe ){
+      const char *zCopy = (const char *)sqlite3_value_text(pLe);
+      pCsr->nLeTerm = sqlite3_value_bytes(pLe);
+      pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1);
+      if( pCsr->zLeTerm==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1);
+      }
+    }
+  }
+
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
+  }
  if( rc==SQLITE_OK ){
    rc = fts5VocabNextMethod(pCursor);
  }
@@ -425,18 +556,36 @@ static int fts5VocabColumnMethod(
  int iCol                        /* Index of column to read value from */
 ){
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
-  switch( iCol ){
-    case 0: /* term */
-      sqlite3_result_text(
-          pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
-      );
-      break;
+  int eDetail = pCsr->pConfig->eDetail;
+  int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType;
+  i64 iVal = 0;

-    default:
-      assert( iCol<4 && iCol>0 );
-      sqlite3_result_int64(pCtx, pCsr->aVal[iCol-1]);
-      break;
+  if( iCol==0 ){
+    sqlite3_result_text(
+        pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
+    );
+  }else if( eType==FTS5_VOCAB_COL ){
+    assert( iCol==1 || iCol==2 || iCol==3 );
+    if( iCol==1 ){
+      if( eDetail!=FTS5_DETAIL_NONE ){
+        const char *z = pCsr->pConfig->azCol[pCsr->iCol];
+        sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
+      }
+    }else if( iCol==2 ){
+      iVal = pCsr->aDoc[pCsr->iCol];
+    }else{
+      iVal = pCsr->aCnt[pCsr->iCol];
+    }
+  }else{
+    assert( iCol==1 || iCol==2 );
+    if( iCol==1 ){
+      iVal = pCsr->aDoc[0];
+    }else{
+      iVal = pCsr->aCnt[0];
+    }
  }
+
+  if( iVal>0 ) sqlite3_result_int64(pCtx, iVal);
  return SQLITE_OK;
 }

@@ -58,6 +58,18 @@
 */
 #define yytestcase(X) testcase(X)

+/*
+** Indicate that sqlite3ParserFree() will never be called with a null
+** pointer.
+*/
+#define YYPARSEFREENOTNULL 1
+
+/*
+** Alternative datatype for the argument to the malloc() routine passed
+** into sqlite3ParserAlloc().  The default is size_t.
+*/
+#define YYMALLOCARGTYPE  u64
+
 } // end %include

 %left OR.
@@ -168,7 +180,3 @@ phrase(A) ::= STRING(Y) star_opt(Z). {

 star_opt(A) ::= STAR. { A = 1; }
 star_opt(A) ::= . { A = 0; }
-
-
-
-
@@ -15,11 +15,22 @@ if {![info exists testdir]} {
 }
 source $testdir/tester.tcl

+ifcapable !fts5 {
+  finish_test
+  return
+}
+
 catch { 
  sqlite3_fts5_may_be_corrupt 0 
  reset_db
 }

+# If SQLITE_ENABLE_FTS5 is not defined, skip this test.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
 proc fts5_test_poslist {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
@@ -28,6 +39,29 @@ proc fts5_test_poslist {cmd} {
  set res
 }

+proc fts5_test_poslist2 {cmd} {
+  set res [list]
+
+  for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
+    $cmd xPhraseForeach $i c o {
+      lappend res $i.$c.$o
+    }
+  }
+
+  #set res
+  sort_poslist $res
+}
+
+proc fts5_test_collist {cmd} {
+  set res [list]
+
+  for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
+    $cmd xPhraseColumnForeach $i c { lappend res $i.$c }
+  }
+
+  set res
+}
+
 proc fts5_test_columnsize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@@ -113,6 +147,8 @@ proc fts5_aux_test_functions {db} {
    fts5_test_columntext
    fts5_test_columntotalsize
    fts5_test_poslist
+    fts5_test_poslist2
+    fts5_test_collist
    fts5_test_tokenize
    fts5_test_rowcount
    fts5_test_all
@@ -178,15 +214,24 @@ proc fts5_rnddoc {n} {
 #   -near N        (NEAR distance. Default 10)
 #   -col  C        (List of column indexes to match against)
 #   -pc   VARNAME  (variable in caller frame to use for phrase numbering)
+#   -dict VARNAME  (array in caller frame to use for synonyms)
 #
 proc nearset {aCol args} {
+
+  # Process the command line options.
+  #
  set O(-near) 10
  set O(-col)  {}
  set O(-pc)   ""
+  set O(-dict) ""

  set nOpt [lsearch -exact $args --]
  if {$nOpt<0} { error "no -- option" }

+  # Set $lPhrase to be a list of phrases. $nPhrase its length.
+  set lPhrase [lrange $args [expr $nOpt+1] end]
+  set nPhrase [llength $lPhrase]
+
  foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
    if {[info exists O($k)]==0} { error "unrecognized option $k" }
    set O($k) $v
@@ -198,9 +243,7 @@ proc nearset {aCol args} {
    upvar $O(-pc) counter
  }

-  # Set $phraselist to be a list of phrases. $nPhrase its length.
-  set phraselist [lrange $args [expr $nOpt+1] end]
-  set nPhrase [llength $phraselist]
+  if {$O(-dict)!=""} { upvar $O(-dict) aDict }

  for {set j 0} {$j < [llength $aCol]} {incr j} {
    for {set i 0} {$i < $nPhrase} {incr i} { 
@@ -208,27 +251,54 @@ proc nearset {aCol args} {
    }
  }

-  set iCol -1
-  foreach col $aCol {
-    incr iCol
-    if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
-    set nToken [llength $col]
+  # Loop through each column of the current row.
+  for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {

-    set iFL [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
-    for { } {$iFL < $nToken} {incr iFL} {
-      for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
-        set B($iPhrase) [list]
-      }
+    # If there is a column filter, test whether this column is excluded. If
+    # so, skip to the next iteration of this loop. Otherwise, set zCol to the
+    # column value and nToken to the number of tokens that comprise it.
+    if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
+    set zCol [lindex $aCol $iCol]
+    set nToken [llength $zCol]
+
+    # Each iteration of the following loop searches a substring of the 
+    # column value for phrase matches. The last token of the substring
+    # is token $iLast of the column value. The first token is:
+    #
+    #   iFirst = ($iLast - $O(-near) - 1)
+    #
+    # where $sz is the length of the phrase being searched for. A phrase 
+    # counts as matching the substring if its first token lies on or before
+    # $iLast and its last token on or after $iFirst.
+    #
+    # For example, if the query is "NEAR(a+b c, 2)" and the column value:
+    #
+    #   "x x x x A B x x C x"
+    #    0 1 2 3 4 5 6 7 8 9"
+    #
+    # when (iLast==8 && iFirst=5) the range will contain both phrases and
+    # so both instances can be added to the output poslists.
+    #
+    set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
+    for { } {$iLast < $nToken} {incr iLast} {
+
+      catch { array unset B }
      
      for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
-        set p [lindex $phraselist $iPhrase]
+        set p [lindex $lPhrase $iPhrase]
        set nPm1 [expr {[llength $p] - 1}]
-        set iFirst [expr $iFL - $O(-near) - [llength $p]]
+        set iFirst [expr $iLast - $O(-near) - [llength $p]]

-        for {set i $iFirst} {$i <= $iFL} {incr i} {
-          if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i }
+        for {set i $iFirst} {$i <= $iLast} {incr i} {
+          set lCand [lrange $zCol $i [expr $i+$nPm1]]
+          set bMatch 1
+          foreach tok $p term $lCand {
+            if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break }
+          }
+          if {$bMatch} { lappend B($iPhrase) $i }
        }
-        if {[llength $B($iPhrase)] == 0} break
+
+        if {![info exists B($iPhrase)]} break
      }

      if {$iPhrase==$nPhrase} {
@@ -252,10 +322,22 @@ proc nearset {aCol args} {
    incr counter
  }

-  #puts $res
+  #puts "$aCol -> $res"
  sort_poslist $res
 }

+proc nearset_match {aDictVar tok term} {
+  if {[string match $tok $term]} { return 1 }
+
+  upvar $aDictVar aDict
+  if {[info exists aDict($tok)]} {
+    foreach s $aDict($tok) {
+      if {[string match $s $term]} { return 1 }
+    }
+  }
+  return 0;
+}
+
 #-------------------------------------------------------------------------
 # Usage:
 #
@@ -327,3 +409,234 @@ proc fts5_tokenize_split {text} {
  set ret
 }

+#-------------------------------------------------------------------------
+#
+proc foreach_detail_mode {prefix script} {
+  set saved $::testprefix
+  foreach d [list full col none] {
+    set s [string map [list %DETAIL% $d] $script]
+    set ::detail $d
+    set ::testprefix "$prefix-$d"
+    reset_db
+    uplevel $s
+    unset ::detail
+  }
+  set ::testprefix $saved
+}
+
+proc detail_check {} {
+  if {$::detail != "none" && $::detail!="full" && $::detail!="col"} {
+    error "not in foreach_detail_mode {...} block"
+  }
+}
+proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
+proc detail_is_col {}  { detail_check ; expr {$::detail == "col" } }
+proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
+
+
+#-------------------------------------------------------------------------
+# Convert a poslist of the type returned by fts5_test_poslist() to a 
+# collist as returned by fts5_test_collist().
+#
+proc fts5_poslist2collist {poslist} {
+  set res [list]
+  foreach h $poslist {
+    regexp {(.*)\.[1234567890]+} $h -> cand
+    lappend res $cand
+  }
+  set res [lsort -command fts5_collist_elem_compare -unique $res]
+  return $res
+}
+
+# Comparison function used by fts5_poslist2collist to sort collist entries.
+proc fts5_collist_elem_compare {a b} {
+  foreach {a1 a2} [split $a .] {}
+  foreach {b1 b2} [split $b .] {}
+
+  if {$a1==$b1} { return [expr $a2 - $b2] }
+  return [expr $a1 - $b1]
+}
+
+
+#--------------------------------------------------------------------------
+# Construct and return a tcl list equivalent to that returned by the SQL
+# query executed against database handle [db]:
+#
+#   SELECT 
+#     rowid, 
+#     fts5_test_poslist($tbl),
+#     fts5_test_collist($tbl) 
+#   FROM $tbl('$expr')
+#   ORDER BY rowid $order;
+#
+proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} {
+
+  # Figure out the set of columns in the FTS5 table. This routine does
+  # not handle tables with UNINDEXED columns, but if it did, it would
+  # have to be here.
+  db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
+
+  set d ""
+  if {$aDictVar != ""} {
+    upvar $aDictVar aDict
+    set d aDict
+  }
+
+  set cols ""
+  foreach e $lCols { append cols ", '$e'" }
+  set tclexpr [db one [subst -novar {
+    SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] )
+  }]]
+
+  set res [list]
+  db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
+    set cols [list]
+    foreach col $lCols { lappend cols $x($col) }
+    
+    set ::pc 0
+    set rowdata [eval $tclexpr]
+    if {$rowdata != ""} { 
+      lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata]
+    }
+  }
+
+  set res
+}
+
+#-------------------------------------------------------------------------
+# Similar to [fts5_query_data], but omit the collist field.
+#
+proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} {
+  set res [list]
+
+  if {$aDictVar!=""} {
+    upvar $aDictVar aDict
+    set dict aDict
+  } else {
+    set dict ""
+  }
+
+  foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
+    lappend res $rowid $poslist
+  }
+  set res
+}
+
+proc fts5_collist_data {expr tbl {order ASC} {aDictVar ""}} {
+  set res [list]
+
+  if {$aDictVar!=""} {
+    upvar $aDictVar aDict
+    set dict aDict
+  } else {
+    set dict ""
+  }
+
+  foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
+    lappend res $rowid $collist
+  }
+  set res
+}
+
+#-------------------------------------------------------------------------
+#
+
+# This command will only work inside a [foreach_detail_mode] block. It tests
+# whether or not expression $expr run on FTS5 table $tbl is supported by
+# the current mode. If so, 1 is returned. If not, 0.
+#
+#   detail=full    (all queries supported)
+#   detail=col     (all but phrase queries and NEAR queries)
+#   detail=none    (all but phrase queries, NEAR queries, and column filters)
+#
+proc fts5_expr_ok {expr tbl} {
+
+  if {![detail_is_full]} {
+    set nearset "nearset_rc"
+    if {[detail_is_col]} { set nearset "nearset_rf" }
+
+    set ::expr_not_ok 0
+    db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
+
+    set cols ""
+    foreach e $lCols { append cols ", '$e'" }
+    set ::pc 0
+    set tclexpr [db one [subst -novar {
+      SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] )
+    }]]
+    eval $tclexpr
+    if {$::expr_not_ok} { return 0 }
+  }
+
+  return 1
+}
+
+# Helper for [fts5_expr_ok]
+proc nearset_rf {aCol args} {
+  set idx [lsearch -exact $args --]
+  if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} {
+    set ::expr_not_ok 1
+  }
+  list
+}
+
+# Helper for [fts5_expr_ok]
+proc nearset_rc {aCol args} {
+  nearset_rf $aCol {*}$args
+  if {[lsearch $args -col]>=0} { 
+    set ::expr_not_ok 1
+  }
+  list
+}
+
+
+#-------------------------------------------------------------------------
+# Code for a simple Tcl tokenizer that supports synonyms at query time.
+#
+proc tclnum_tokenize {mode tflags text} {
+  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+    sqlite3_fts5_token $w $iStart $iEnd
+    if {$tflags == $mode && [info exists ::tclnum_syn($w)]} {
+      foreach s $::tclnum_syn($w)  { sqlite3_fts5_token -colo $s $iStart $iEnd }
+    }
+  }
+}
+
+proc tclnum_create {args} {
+  set mode query
+  if {[llength $args]} {
+    set mode [lindex $args 0]
+  }
+  if {$mode != "query" && $mode != "document"} { error "bad mode: $mode" }
+  return [list tclnum_tokenize $mode]
+}
+
+proc fts5_tclnum_register {db} {
+  foreach SYNDICT {
+    {zero  0}
+    {one   1 i}
+    {two   2 ii}
+    {three 3 iii}
+    {four  4 iv}
+    {five  5 v}
+    {six   6 vi}
+    {seven 7 vii}
+    {eight 8 viii}
+    {nine  9 ix}
+
+    {a1 a2 a3 a4 a5 a6 a7 a8 a9}
+    {b1 b2 b3 b4 b5 b6 b7 b8 b9}
+    {c1 c2 c3 c4 c5 c6 c7 c8 c9}
+  } {
+    foreach s $SYNDICT {
+      set o [list]
+      foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
+      set ::tclnum_syn($s) $o
+    }
+  }
+  sqlite3_fts5_create_tokenizer db tclnum tclnum_create
+}
+#
+# End of tokenizer code.
+#-------------------------------------------------------------------------
+
@@ -15,12 +15,14 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aa

-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
 }

+foreach_detail_mode $::testprefix {
+
 do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
  SELECT name, sql FROM sqlite_master;
@@ -41,9 +43,9 @@ do_execsql_test 1.1 {

 #-------------------------------------------------------------------------
 #
-reset_db
+
 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
 }
 do_execsql_test 2.1 {
  INSERT INTO t1 VALUES('a b c', 'd e f');
@@ -66,11 +68,12 @@ do_execsql_test 2.4 {
  INSERT INTO t1(t1) VALUES('integrity-check');
 }

+
 #-------------------------------------------------------------------------
 #
 reset_db
 do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
 }
 foreach {i x y} {
   1  {g f d b f} {h h e i a}
@@ -93,7 +96,7 @@ foreach {i x y} {
 #
 reset_db
 do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -117,7 +120,7 @@ foreach {i x y} {
 #
 reset_db
 do_execsql_test 5.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -141,7 +144,7 @@ foreach {i x y} {
 #
 reset_db
 do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }

@@ -159,9 +162,20 @@ do_execsql_test 6.3 {
 }

 do_execsql_test 6.4 {
+  REPLACE INTO t1(x, y) VALUES('x y z', 'x y z');
+}
+
+do_execsql_test 6.5 {
  INSERT INTO t1(t1) VALUES('integrity-check') 
 }

+do_execsql_test 6.6 {
+  SELECT rowid, * FROM t1;
+} {
+  22 {l l l} {l l l}
+  23 {x y z} {x y z}
+}
+
 #-------------------------------------------------------------------------
 #
 reset_db
@@ -265,7 +279,7 @@ for {set i 1} {$i <= 10} {incr i} {
 #
 reset_db
 do_execsql_test 10.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
 }
 set d10 {
   1  {g f d b f} {h h e i a}
@@ -298,19 +312,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 #-------------------------------------------------------------------------
 #
 do_catchsql_test 11.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
 } {1 {reserved fts5 column name: rank}}
 do_catchsql_test 11.2 {
-  CREATE VIRTUAL TABLE rank USING fts5(a, b, c);
+  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
 } {1 {reserved fts5 table name: rank}}
 do_catchsql_test 11.3 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
 } {1 {reserved fts5 column name: rowid}}

 #-------------------------------------------------------------------------
 #
 do_execsql_test 12.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x,y);
+  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
 } {}

 do_catchsql_test 12.2 {
@@ -326,7 +340,7 @@ do_test 12.3 {
 #
 reset_db
 do_execsql_test 13.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
  INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
 } {}

@@ -350,7 +364,7 @@ do_execsql_test 13.6 {
 #
 reset_db
 do_execsql_test 14.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  WITH d(x,y) AS (
    SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
@@ -360,6 +374,10 @@ do_execsql_test 14.1 {
  INSERT INTO t1 SELECT * FROM d LIMIT 200;
 }

+do_execsql_test 15.x {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
 do_test 14.2 {
  set nRow 0
  db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } {
@@ -406,11 +424,11 @@ do_execsql_test 16.1 {
 }

 proc funk {} {
+  db eval { UPDATE n1_config SET v=50 WHERE k='version' }
  set fd [db incrblob main n1_data block 10]
  fconfigure $fd -encoding binary -translation binary
  puts -nonewline $fd "\x44\x45"
  close $fd
-  db eval { UPDATE n1_config SET v=50 WHERE k='version' }
 }
 db func funk funk

@@ -422,7 +440,7 @@ do_catchsql_test 16.2 {
 #
 reset_db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE b2 USING fts5(x);
+  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
  INSERT INTO b2 VALUES('a');
  INSERT INTO b2 VALUES('b');
  INSERT INTO b2 VALUES('c');
@@ -436,18 +454,20 @@ do_test 17.2 {
  set res
 } {{a b c} {a b c} {a b c}}

-reset_db
-do_execsql_test 18.1 {
-  CREATE VIRTUAL TABLE c2 USING fts5(x, y);
-  INSERT INTO c2 VALUES('x x x', 'x x x');
-  SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
-} {1}
+if {[string match n* %DETAIL%]==0} {
+  reset_db
+  do_execsql_test 17.3 {
+    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
+    INSERT INTO c2 VALUES('x x x', 'x x x');
+    SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
+  } {1}
+}

 #-------------------------------------------------------------------------
 #
 reset_db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE uio USING fts5(ttt);
+  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
  INSERT INTO uio VALUES(NULL);
  INSERT INTO uio SELECT NULL FROM uio;
  INSERT INTO uio SELECT NULL FROM uio;
@@ -494,8 +514,8 @@ do_execsql_test 17.9 {
 #--------------------------------------------------------------------
 #
 do_execsql_test 18.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
-  CREATE VIRTUAL TABLE t2 USING fts5(c, d);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
  INSERT INTO t1 VALUES('abc*', NULL);
  INSERT INTO t2 VALUES(1, 'abcdefg');
 }
@@ -511,7 +531,7 @@ do_execsql_test 18.3 {
 #
 reset_db
 do_execsql_test 19.0 {
-  CREATE VIRTUAL TABLE temp.t1 USING fts5(x);
+  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
  INSERT INTO t1 VALUES('x y z');
  INSERT INTO t1 VALUES('w x 1');
  SELECT rowid FROM t1 WHERE t1 MATCH 'x';
@@ -522,7 +542,7 @@ do_execsql_test 19.0 {
 #
 reset_db
 do_execsql_test 20.0 {
-  CREATE VIRTUAL TABLE temp.tmp USING fts5(x);
+  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
 }
 set ::ids [list \
  0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
@@ -534,6 +554,7 @@ do_test 20.1 {
  execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
 } $::ids

+}


 finish_test
@@ -22,8 +22,10 @@ ifcapable !fts5 {
  return
 }

+foreach_detail_mode $testprefix {
+
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
  INSERT INTO t1 VALUES('hello', 'world');
  INSERT INTO t1 VALUES('one two', 'three four');
  INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five');
@@ -57,7 +59,7 @@ do_execsql_test 1.6 {

 reset_db
 do_execsql_test 2.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  INSERT INTO t1 VALUES('one');
  INSERT INTO t1 VALUES('two');
@@ -159,7 +161,7 @@ foreach {tn expr res} {
 #

 do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE s1 USING fts5(x);
+  CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%);
 }
 foreach {tn doc} [list \
  1 [string repeat {a x } 1500000]       \
@@ -172,8 +174,14 @@ do_execsql_test 4.3 {
  SELECT rowid FROM s1 WHERE s1 MATCH 'x'
 } {1 2}

-do_execsql_test 4.4 {
-  SELECT rowid FROM s1 WHERE s1 MATCH '"a x"'
+if {[detail_is_full]} {
+  do_execsql_test 4.4 {
+    SELECT rowid FROM s1 WHERE s1 MATCH '"a x"'
+  } {1 2}
+}
+
+do_execsql_test 4.5 {
+  SELECT rowid FROM s1 WHERE s1 MATCH 'a x'
 } {1 2}

 #-------------------------------------------------------------------------
@@ -182,7 +190,7 @@ do_execsql_test 4.4 {
 # (L-2) is larger than it.
 #
 do_execsql_test 5.0 {
-  CREATE VIRTUAL TABLE s2 USING fts5(x);
+  CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%);
  INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
  INSERT INTO s2(s2, rank) VALUES('automerge', 0);
 }
@@ -222,7 +230,7 @@ do_test 5.2 {
 do_test 5.3 {
  execsql {
    DROP TABLE s2;
-    CREATE VIRTUAL TABLE s2 USING fts5(x);
+    CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%);
    INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
    INSERT INTO s2(s2, rank) VALUES('automerge', 0);
  }
@@ -241,7 +249,7 @@ do_test 5.4 {
 #-------------------------------------------------------------------------
 #
 do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE s3 USING fts5(x);
+  CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%);
  BEGIN;
    INSERT INTO s3 VALUES('a b c');
    INSERT INTO s3 VALUES('A B C');
@@ -276,13 +284,13 @@ do_test 6.4 {
 #-------------------------------------------------------------------------
 #
 set doc [string repeat "a b c " 500]
-breakpoint
 do_execsql_test 7.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(x);
+  CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%);
  INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
  INSERT INTO x1 VALUES($doc);
 }

+} ;# foreach_detail_mode...


 finish_test
@@ -22,6 +22,8 @@ ifcapable !fts5 {
  return
 }

+foreach_detail_mode $testprefix {
+
 set data {
    0   {p o q e z k z p n f y u z y n y}   {l o o l v v k}
    1   {p k h h p y l l h i p v n}         {p p l u r i f a j g e r r x w}
@@ -125,80 +127,21 @@ set data {
    99  {r c v w i v h a t a c v c r e}     {h h u m g o f b a e o}
 }

-# Argument $expr is an FTS5 match expression designed to be executed against
-# an FTS5 table with the following schema:
-# 
-#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
-#
-# Assuming the table contains the same records as stored int the global 
-# $::data array (see above), this function returns a list containing one
-# element for each match in the dataset. The elements are themselves lists
-# formatted as follows:
-#
-#   <rowid> {<phrase 0 matches> <phrase 1 matches>...}
-#
-# where each <phrase X matches> element is a list of phrase matches in the
-# same form as returned by auxiliary scalar function fts5_test().
-#
-proc matchdata {bPos expr {bAsc 1}} {
-
-  set tclexpr [db one {
-    SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y')
-  }]
-  set res [list]
-
-  #puts $tclexpr
-  foreach {id x y} $::data {
-    set cols [list $x $y]
-    set ::pc 0
-    #set hits [lsort -command instcompare [eval $tclexpr]]
-    set hits [eval $tclexpr]
-    if {[llength $hits]>0} {
-      if {$bPos} {
-        lappend res [list $id $hits]
-      } else {
-        lappend res $id
-      }
-    }
-  }
-
-  if {$bAsc} {
-    set res [lsort -integer -increasing -index 0 $res]
-  } else {
-    set res [lsort -integer -decreasing -index 0 $res]
-  }
-
-  return [concat {*}$res]
-}
-
-#
-# End of test code
-#-------------------------------------------------------------------------
-
-proc fts5_test_poslist {cmd} {
-  set res [list]
-  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
-    lappend res [string map {{ } .} [$cmd xInst $i]]
-  }
-  set res
-}
-
-
 foreach {tn2 sql} {
  1  {}
  2  {BEGIN}
 } {
  reset_db
-  sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist
+  fts5_aux_test_functions db

-  do_execsql_test 1.0 {
-    CREATE VIRTUAL TABLE xx USING fts5(x,y);
+  do_execsql_test 1.$tn2.0 {
+    CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%);
    INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
  }

  execsql $sql

-  do_test $tn2.1.1 {
+  do_test 1.$tn2.1.1 {
    foreach {id x y} $data {
      execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
    }
@@ -207,153 +150,130 @@ foreach {tn2 sql} {


  #-------------------------------------------------------------------------
-  # Test phrase queries.
  #
-  foreach {tn phrase} {
-    1 "o"
-    2 "b q"
-    3 "e a e"
-    4 "m d g q q b k b w f q q p p"
-    5 "l o o l v v k"
-    6 "a"
-    7 "b"
-    8 "c"
-    9 "no"
-    10 "L O O L V V K"
-  } {
-    set expr "\"$phrase\""
-    set res [matchdata 1 $expr]
-
-    do_execsql_test $tn2.1.2.$tn.[llength $res] { 
-      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-    } $res
+  do_execsql_test 1.$tn2.integrity {
+    INSERT INTO xx(xx) VALUES('integrity-check');
  }

  #-------------------------------------------------------------------------
-  # Test some AND and OR queries.
  #
  foreach {tn expr} {
    1.1 "a   AND b"
-    1.2 "a+b AND c"
-    1.3 "d+c AND u"
-    1.4 "d+c AND u+d"
+    1.2 "a   OR b"
+    1.3 "o"
+    1.4 "b q"
+    1.5 "e a e"
+    1.6 "m d g q q b k b w f q q p p"
+    1.7 "l o o l v v k"
+    1.8 "a"
+    1.9 "b"
+    1.10 "c"
+    1.11 "no"
+    1.12 "L O O L V V K"
+    1.13 "a AND b AND c"
+    1.14 "x:a"

-    2.1 "a   OR b"
-    2.2 "a+b OR c"
-    2.3 "d+c OR u"
-    2.4 "d+c OR u+d"
+    2.1 "x:a"
+    2.2 "y:a"
+    2.3 "x:b"
+    2.4 "y:b"

-    3.1 { a AND b AND c }
+    3.1 "{x}:a"
+    3.2 "{y}:a"
+    3.3 "{x}:b"
+    3.4 "{y}:b"
+
+    4.1 "{x y}:a"
+    4.2 "{y x}:a"
+    4.3 "{x x}:b"
+    4.4 "{y y}:b"
+
+    5.1 {{"x" "y"}:a}
+    5.2 {{"y" x}:a}
+    5.3 {{x "x"}:b}
+    5.4 {{"y" y}:b}
+
+    6.1 "b + q"
+    6.2 "e + a + e"
+    6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p"
+    6.4 "l + o + o + l + v + v + k"
+    6.5 "L + O + O + L + V + V + K"
+
+    7.1 "a+b AND c"
+    7.2 "d+c AND u"
+    7.3 "d+c AND u+d"
+    7.4 "a+b OR c"
+    7.5 "d+c OR u"
+    7.6 "d+c OR u+d"
+
+    8.1 "NEAR(a b)"
+    8.2 "NEAR(r c)"
+    8.2 { NEAR(r c, 5) }
+    8.3 { NEAR(r c, 3) }
+    8.4 { NEAR(r c, 2) }
+    8.5 { NEAR(r c, 0) }
+    8.6 { NEAR(a b c) }
+    8.7 { NEAR(a b c, 8) }
+    8.8  { x : NEAR(r c) }
+    8.9  { y : NEAR(r c) }
+
+    9.1 { NEAR(r c) }
+    9.2 { NEAR(r c, 5) }
+    9.3 { NEAR(r c, 3) }
+    9.4 { NEAR(r c, 2) }
+    9.5 { NEAR(r c, 0) }
+    9.6 { NEAR(a b c) }
+    9.7 { NEAR(a b c, 8) }
+    9.8  { x : NEAR(r c) }
+    9.9  { y : NEAR(r c) }
+    9.10 { x : "r c" }
+    9.11 { y : "r c" }
+    9.12 { a AND b }
+    9.13 { a AND b AND c }
+    9.14a { a }
+    9.14b { a OR b }
+    9.15 { a OR b AND c }
+    9.16 { c AND b OR a }
+    9.17 { c AND (b OR a) }
+    9.18 { c NOT (b OR a) }
+    9.19 { (c NOT b) OR (a AND d) }
  } {
-    set res [matchdata 1 $expr]
-    do_execsql_test $tn2.2.$tn.[llength $res] { 
-      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-    } $res
-  }

-  #-------------------------------------------------------------------------
-  # Queries on a specific column.
-  #
-  foreach {tn expr} {
-    1.1 "x:a"
-    1.2 "y:a"
-    1.3 "x:b"
-    1.4 "y:b"
-    2.1 "{x}:a"
-    2.2 "{y}:a"
-    2.3 "{x}:b"
-    2.4 "{y}:b"
-
-    3.1 "{x y}:a"
-    3.2 "{y x}:a"
-    3.3 "{x x}:b"
-    3.4 "{y y}:b"
-
-    4.1 {{"x" "y"}:a}
-    4.2 {{"y" x}:a}
-    4.3 {{x "x"}:b}
-    4.4 {{"y" y}:b}
-  } {
-    set res [matchdata 1 $expr]
-    do_execsql_test $tn2.3.$tn.[llength $res] { 
-      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-    } $res
-  }
-
-  #-------------------------------------------------------------------------
-  # Some NEAR queries.
-  #
-  foreach {tn expr} {
-    1 "NEAR(a b)"
-    2 "NEAR(r c)"
-    2 { NEAR(r c, 5) }
-    3 { NEAR(r c, 3) }
-    4 { NEAR(r c, 2) }
-    5 { NEAR(r c, 0) }
-    6 { NEAR(a b c) }
-    7 { NEAR(a b c, 8) }
-    8  { x : NEAR(r c) }
-    9  { y : NEAR(r c) }
-  } {
-    set res [matchdata 1 $expr]
-    do_execsql_test $tn2.4.1.$tn.[llength $res] { 
-      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-    } $res
-  }
-
-  do_test $tn2.4.1  { nearset {{a b c}} -- a } {0.0.0}
-  do_test $tn2.4.2  { nearset {{a b c}} -- c } {0.0.2}
-
-  foreach {tn expr tclexpr} {
-    1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
-  } {
-    do_execsql_test $tn2.5.$tn {
-      SELECT fts5_expr_tcl($expr, 'N $x')
-    } [list $tclexpr]
-  }
-
-  #-------------------------------------------------------------------------
-  #
-  do_execsql_test $tn2.6.integrity {
-    INSERT INTO xx(xx) VALUES('integrity-check');
-  }
-  #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r}
-  foreach {bAsc sql} {
-    1 {SELECT rowid FROM xx WHERE xx MATCH $expr}
-    0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC}
-  } {
-    foreach {tn expr} {
-      0.1 x
-      1 { NEAR(r c) }
-      2 { NEAR(r c, 5) }
-      3 { NEAR(r c, 3) }
-      4 { NEAR(r c, 2) }
-      5 { NEAR(r c, 0) }
-      6 { NEAR(a b c) }
-      7 { NEAR(a b c, 8) }
-      8  { x : NEAR(r c) }
-      9  { y : NEAR(r c) }
-      10 { x : "r c" }
-      11 { y : "r c" }
-      12 { a AND b }
-      13 { a AND b AND c }
-      14a { a }
-      14b { a OR b }
-      15 { a OR b AND c }
-      16 { c AND b OR a }
-      17 { c AND (b OR a) }
-      18 { c NOT (b OR a) }
-      19 { c NOT b OR a AND d }
-    } {
-      set res [matchdata 0 $expr $bAsc]
-      do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res
+    if {[fts5_expr_ok $expr xx]==0} {
+      do_test 1.$tn2.$tn.OMITTED { list } [list]
+      continue
    }
+
+    set res [fts5_query_data $expr xx]
+    do_execsql_test 1.$tn2.$tn.[llength $res].asc {
+      SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
+      FROM xx WHERE xx match $expr
+    } $res
+
+    set res [fts5_query_data $expr xx DESC]
+    do_execsql_test 1.$tn2.$tn.[llength $res].desc {
+      SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
+      FROM xx WHERE xx match $expr ORDER BY 1 DESC
+    } $res
  }
 }

-do_execsql_test 3.1 {
+}
+
+do_execsql_test 2.1 {
  SELECT fts5_expr_tcl('a AND b');
 } {{AND [nearset -- {a}] [nearset -- {b}]}}

+do_test 2.2.1  { nearset {{a b c}} -- a } {0.0.0}
+do_test 2.2.2  { nearset {{a b c}} -- c } {0.0.2}
+
+foreach {tn expr tclexpr} {
+  1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
+} {
+  do_execsql_test 2.3.$tn {
+    SELECT fts5_expr_tcl($expr, 'N $x')
+  } [list $tclexpr]
+}
+
 finish_test

@@ -11,6 +11,8 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is testing the FTS5 module.
 #
+# More specifically, the focus is on testing prefix queries, both with and
+# without prefix indexes.
 #

 source [file join [file dirname [info script]] fts5_common.tcl]
@@ -22,8 +24,10 @@ ifcapable !fts5 {
  return
 }

+foreach_detail_mode $testprefix {
+
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE yy USING fts5(x, y);
+  CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%);
  INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching');
  INSERT INTO yy VALUES('indices (or all  matching', 'values if -inline is');
  INSERT INTO yy VALUES('specified as  well.) If', 'indices are returned, the');
@@ -53,23 +57,23 @@ foreach {tn match res} {

 foreach {T create} {
  2 {
-    CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }
  
  3 {
-    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }

  4 {
-    CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    BEGIN;
  }
  
  5 {
-    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    BEGIN;
  }
@@ -235,5 +239,7 @@ foreach {T create} {
  catchsql COMMIT
 }

+}
+
 finish_test

@@ -22,8 +22,10 @@ ifcapable !fts5 {
  return
 }

+foreach_detail_mode $testprefix {
+
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }

@@ -55,7 +57,7 @@ fts5_aux_test_functions db
 #-------------------------------------------------------------------------
 # 
 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x, y);
+  CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%);
  INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e');
  INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p');
 }
@@ -76,31 +78,35 @@ do_execsql_test 2.2 {
  2 {1.0.2 1.0.10}
 }

-do_execsql_test 2.3 {
-  SELECT rowid, fts5_test_poslist(t2) FROM t2 
-  WHERE t2 MATCH 'y:o' ORDER BY rowid;
-} {
-  1 {0.1.3 0.1.7}
+if {[detail_is_full]} {
+  do_execsql_test 2.3 {
+    SELECT rowid, fts5_test_poslist(t2) FROM t2 
+      WHERE t2 MATCH 'y:o' ORDER BY rowid;
+  } {
+    1 {0.1.3 0.1.7}
+  }
 }

 #-------------------------------------------------------------------------
 # 
 do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE t3 USING fts5(x, y);
+  CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%);
  INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t');
  INSERT INTO t3 VALUES( 'r c', '');
 }

-do_execsql_test 3.1 {
-  SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)';
-} {
-  1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15}
-}
+if {[detail_is_full]} {
+  do_execsql_test 3.1 {
+    SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)';
+  } {
+    1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15}
+  }

-do_execsql_test 3.2 {
-  SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)';
-} {
-  2 {0.0.0 1.0.1}
+  do_execsql_test 3.2 {
+    SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)';
+  } {
+    2 {0.0.0 1.0.1}
+  }
 }

 do_execsql_test 3.3 {
@@ -116,7 +122,7 @@ do_execsql_test 3.3 {
 #-------------------------------------------------------------------------
 # 
 do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
+  CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%);
  INSERT INTO t4 
  VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
 }
@@ -134,7 +140,7 @@ reset_db
 fts5_aux_test_functions db

 do_execsql_test 5.1 {
-  CREATE VIRTUAL TABLE t5 USING fts5(x, y);
+  CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%);
  INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
  INSERT INTO t5 VALUES('', 'a');
  INSERT INTO t5 VALUES('a', '');
@@ -182,7 +188,7 @@ do_execsql_test 5.5 {
 reset_db
 fts5_aux_test_functions db
 do_execsql_test 6.1 {
-  CREATE VIRTUAL TABLE t6 USING fts5(x, y);
+  CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%);
  INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
  INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
 }
@@ -200,7 +206,7 @@ do_execsql_test 6.2 {
 reset_db
 fts5_aux_test_functions db
 do_execsql_test 7.1 {
-  CREATE VIRTUAL TABLE t7 USING fts5(x, y);
+  CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%);
 }
 do_test 7.2 {
  foreach {x y} {
@@ -240,7 +246,7 @@ do_execsql_test 7.4 {
 #-------------------------------------------------------------------------
 #
 do_test 8.1 {
-  execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y) }
+  execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) }
  foreach {rowid x y} {
     0 {A o}   {o o o C o o o o o o o o}
     1 {o o B} {o o o C C o o o o o o o}
@@ -300,5 +306,7 @@ foreach {tn q cnt} {
  } $cnt
 }

+}
+
 finish_test

@@ -24,9 +24,10 @@ ifcapable !fts5 {
  return
 }

+foreach_detail_mode $testprefix {

 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
 }

 proc do_snippet_test {tn doc match res} {
@@ -111,34 +112,37 @@ foreach {tn doc res} {
  do_snippet_test 1.$tn $doc X $res
 }

-foreach {tn doc res} {
-  1.1 {X Y o o o o o} {[X Y] o o o o o}
-  1.2 {o X Y o o o o} {o [X Y] o o o o}
-  1.3 {o o X Y o o o} {o o [X Y] o o o}
-  1.4 {o o o X Y o o} {o o o [X Y] o o}
-  1.5 {o o o o X Y o} {o o o o [X Y] o}
-  1.6 {o o o o o X Y} {o o o o o [X Y]}
+if {[detail_is_full]} {
+  foreach {tn doc res} {
+    1.1 {X Y o o o o o} {[X Y] o o o o o}
+    1.2 {o X Y o o o o} {o [X Y] o o o o}
+    1.3 {o o X Y o o o} {o o [X Y] o o o}
+    1.4 {o o o X Y o o} {o o o [X Y] o o}
+    1.5 {o o o o X Y o} {o o o o [X Y] o}
+    1.6 {o o o o o X Y} {o o o o o [X Y]}

-  2.1 {X Y o o o o o o} {[X Y] o o o o o...}
-  2.2 {o X Y o o o o o} {o [X Y] o o o o...}
-  2.3 {o o X Y o o o o} {o o [X Y] o o o...}
-  2.4 {o o o X Y o o o} {...o o [X Y] o o o}
-  2.5 {o o o o X Y o o} {...o o o [X Y] o o}
-  2.6 {o o o o o X Y o} {...o o o o [X Y] o}
-  2.7 {o o o o o o X Y} {...o o o o o [X Y]}
+    2.1 {X Y o o o o o o} {[X Y] o o o o o...}
+    2.2 {o X Y o o o o o} {o [X Y] o o o o...}
+    2.3 {o o X Y o o o o} {o o [X Y] o o o...}
+    2.4 {o o o X Y o o o} {...o o [X Y] o o o}
+    2.5 {o o o o X Y o o} {...o o o [X Y] o o}
+    2.6 {o o o o o X Y o} {...o o o o [X Y] o}
+    2.7 {o o o o o o X Y} {...o o o o o [X Y]}

-  3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
-  3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
-  3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
-  3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
-  3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
-  3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
-  3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
-  3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
-
-} {
-  do_snippet_test 2.$tn $doc "X + Y" $res
+    3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
+    3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
+    3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
+    3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
+    3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
+    3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
+    3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
+    3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
+  } {
+    do_snippet_test 2.$tn $doc "X + Y" $res
+  }
 }

+} ;# foreach_detail_mode 
+
 finish_test

@@ -33,8 +33,10 @@ ifcapable !fts5 {
 #      ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC]
 #

+foreach_detail_mode $testprefix {
+
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, z);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, z, detail=%DETAIL%);
 }

 do_test 1.1 {
@@ -119,19 +121,24 @@ foreach {tn expr} {
  2.3 c
  2.4 d

-  2.5 {"m m"}
-  2.6 {e + s}
-
  3.0 {a AND b}
  3.1 {a OR b}
  3.2 {b OR c AND d}
-  3.3 {NEAR(c d)}
 } {
  do_fts5ag_test $tn $expr
-
-  if {[set_test_counter errors]} break
 }

+if {[detail_is_full]} {
+  foreach {tn expr} {
+    4.1 {"m m"}
+    4.2 {e + s}
+    4.3 {NEAR(c d)}
+  } {
+    do_fts5ag_test $tn $expr
+  }
+}
+
+} ;# foreach_detail_mode


 finish_test
@@ -21,12 +21,16 @@ ifcapable !fts5 {
  return
 }

+foreach_detail_mode $testprefix {
+
 #-------------------------------------------------------------------------
 # This file contains tests for very large doclists.
 #

+set Y [list]
+set W [list]
 do_test 1.0 {
-  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
+  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%) }
  execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) }
  set v {w w w w w w w w w w w w w w w w w w w w}
  execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) }
@@ -70,7 +74,12 @@ do_test 1.4 {
  set nRead [reads]
  execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' }
  set nReadX [expr [reads] - $nRead]
-  expr $nReadX>1000
+  #puts -nonewline "(nReadX=$nReadX)"
+  if {[detail_is_full]} { set expect 1000 }
+  if {[detail_is_col]}  { set expect 250 }
+  if {[detail_is_none]} { set expect 80 }
+
+  expr $nReadX>$expect
 } {1}

 do_test 1.5 {
@@ -87,17 +96,22 @@ foreach {tn q res} "
  3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' }  [list $W]
  4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' }  [list $Y]
 " {
+  if {[detail_is_full]==0 && ($tn==1 || $tn==2)} continue
+
+  if {[detail_is_full]} { set ratio 8 }
+  if {[detail_is_col]}  { set ratio 4 }
+  if {[detail_is_none]} { set ratio 2 }

  do_test 1.6.$tn.1 {
    set n [execsql_reads $q]
    #puts -nonewline "(n=$n nReadX=$nReadX)"
-    expr {$n < ($nReadX / 8)}
+    expr {$n < ($nReadX / $ratio)}
  } {1}

  do_test 1.6.$tn.2 {
    set n [execsql_reads "$q ORDER BY rowid DESC"]
    #puts -nonewline "(n=$n nReadX=$nReadX)"
-    expr {$n < ($nReadX / 8)}
+    expr {$n < ($nReadX / $ratio)}
  } {1}

  do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res]
@@ -109,21 +123,26 @@ foreach {tn q res} "
 # number of pages loaded from disk.
 #
 foreach {tn fraction tail cnt} {
-  1 0.6 {rowid > 5000} 5000
-  2 0.2 {rowid > 9000} 1000
-  3 0.2 {rowid < 1000}  999
-  4 0.2 {rowid BETWEEN 4000 AND 5000}  1001
-  5 0.6 {rowid >= 5000} 5001
-  6 0.2 {rowid >= 9000} 1001
-  7 0.2 {rowid <= 1000} 1000
-  8 0.6 {rowid > '5000'} 5000
-  9 0.2 {rowid > '9000'} 1000
+  1  0.6 {rowid > 5000} 5000
+  2  0.2 {rowid > 9000} 1000
+  3  0.2 {rowid < 1000}  999
+  4  0.2 {rowid BETWEEN 4000 AND 5000}  1001
+  5  0.6 {rowid >= 5000} 5001
+  6  0.2 {rowid >= 9000} 1001
+  7  0.2 {rowid <= 1000} 1000
+  8  0.6 {rowid > '5000'} 5000
+  9  0.2 {rowid > '9000'} 1000
  10 0.1 {rowid = 444} 1
 } {
  set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail"
  set n [execsql_reads $q]
  set ret [llength [execsql $q]]

+  # Because the position lists for 'x' are quite long in this db, the 
+  # advantage is a bit smaller in detail=none mode. Update $fraction to 
+  # reflect this.
+  if {[detail_is_none] && $fraction<0.5} { set fraction [expr $fraction*2] }
+
  do_test "1.7.$tn.asc.(n=$n ret=$ret)" {
    expr {$n < ($fraction*$nReadX) && $ret==$cnt}
  } {1}
@@ -143,6 +162,7 @@ do_execsql_test 1.8.2 {
  SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text';
 } {10000}

+} ;# foreach_detail_mode

 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}

@@ -23,8 +23,10 @@ ifcapable !fts5 {
  return
 }

+foreach_detail_mode $testprefix {
+
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%);
 } {}

 do_execsql_test 1.1 {
@@ -49,6 +51,7 @@ do_execsql_test 1.1 {
 do_execsql_test 1.2 {
  INSERT INTO t1(t1) VALUES('integrity-check');
 }
+}


 finish_test
@@ -23,8 +23,10 @@ ifcapable !fts5 {
  return
 }

+foreach_detail_mode $testprefix {
+
 do_execsql_test 1.1 {
-  CREATE VIRTUAL TABLE ft1 USING fts5(x);
+  CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%);
  INSERT INTO ft1 VALUES('i d d a g i b g d d');
  INSERT INTO ft1 VALUES('h d b j c c g a c a');
  INSERT INTO ft1 VALUES('e j a e f h b f h h');
@@ -35,6 +37,9 @@ do_execsql_test 1.1 {
  INSERT INTO ft1 VALUES('i c c f a d g h j e');
  INSERT INTO ft1 VALUES('i d i g c d c h b f');
  INSERT INTO ft1 VALUES('g d a e h a b c f j');
+
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, detail=%DETAIL%);
+  INSERT INTO ft2 VALUES('a b c d e f g h i j');
 }

 do_execsql_test 1.2 {
@@ -49,19 +54,6 @@ do_execsql_test 1.2 {
 }

 do_execsql_test 1.3 {
-  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d';
-} {
-  {[h d] b j c c g a c a}
-  {j f [h d] g h i b d f} 
-}
-
-do_execsql_test 1.4 {
-  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d';
-} {
-  {i [d d] a g i b g [d d]}
-}
-
-do_execsql_test 1.5 {
  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e e e'
 } {
  {[e] j a [e] f h b f h h}
@@ -72,57 +64,71 @@ do_execsql_test 1.5 {
  {g d a [e] h a b c f j}
 }

-do_execsql_test 1.6 {
-  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d';
-} {
-  {i [d d] a g i b g [d d]}
-}
-
-do_execsql_test 2.1 {
-  CREATE VIRTUAL TABLE ft2 USING fts5(x);
-  INSERT INTO ft2 VALUES('a b c d e f g h i j');
-}
-
-do_execsql_test 2.2 {
-  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e'
-} {{a [b c d e] f g h i j}}
-
-do_execsql_test 2.3 {
-  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g'
-} {
-  {a [b c d] [e f g] h i j}
-}
-
-do_execsql_test 2.4 {
-  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c'
-} {
-  {a [b c d] e f g h i j}
-}
-
-do_execsql_test 2.5 {
-  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e'
-} {
-  {a [b c d e] f g h i j}
-}
-
-do_execsql_test 2.6.1 {
+do_execsql_test 1.4 {
  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d'
 } {
  {a b c [d] e [f] g h i j}
 }

-do_execsql_test 2.6.2 {
+do_execsql_test 1.5 {
  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f'
 } {
  {a b c [d] e [f] g h i j}
 }

+#-------------------------------------------------------------------------
+# Tests below this point require detail=full.
+#-------------------------------------------------------------------------
+if {[detail_is_full]==0} continue
+
+
+do_execsql_test 2.1 {
+  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d';
+} {
+  {[h d] b j c c g a c a}
+  {j f [h d] g h i b d f} 
+}
+
+do_execsql_test 2.2 {
+  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d';
+} {
+  {i [d d] a g i b g [d d]}
+}
+
+do_execsql_test 2.3 {
+  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d';
+} {
+  {i [d d] a g i b g [d d]}
+}
+
+do_execsql_test 2.4 {
+  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e'
+} {{a [b c d e] f g h i j}}
+
+do_execsql_test 2.5 {
+  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g'
+} {
+  {a [b c d] [e f g] h i j}
+}
+
+do_execsql_test 2.6 {
+  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c'
+} {
+  {a [b c d] e f g h i j}
+}
+
+do_execsql_test 2.7 {
+  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e'
+} {
+  {a [b c d e] f g h i j}
+}
+
 #-------------------------------------------------------------------------
 # The example from the docs.
 #
 do_execsql_test 3.1 {
  -- Assuming this:
-  CREATE VIRTUAL TABLE ft USING fts5(a);
+  CREATE VIRTUAL TABLE ft USING fts5(a, detail=%DETAIL%);
  INSERT INTO ft VALUES('a b c x c d e');
  INSERT INTO ft VALUES('a b c c d e');
  INSERT INTO ft VALUES('a b c d e');
@@ -138,6 +144,7 @@ do_execsql_test 3.1 {
  {[a b c d e]}
 }

+}

 finish_test

@@ -23,8 +23,10 @@ ifcapable !fts5 {
  return
 }

+foreach_detail_mode $testprefix {
+
 do_execsql_test 1.1 {
-  CREATE VIRTUAL TABLE ft1 USING fts5(x);
+  CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%);
  SELECT * FROM ft1_config;
 } {version 4}

@@ -83,7 +85,7 @@ foreach {tn defn} {
 #

 do_execsql_test 3.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
  INSERT INTO t1 VALUES('q w e r t y');
  INSERT INTO t1 VALUES('y t r e w q');
 }
@@ -122,11 +124,13 @@ do_execsql_test 3.4.1 {
  {{0 0 5}} 
 }

-do_execsql_test 3.4.2 {
-  SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w'
-} {
-  {{1 0 1}}
-  {{0 0 2} {1 0 4}} 
+if {[detail_is_full]} {
+  do_execsql_test 3.4.2 {
+    SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w'
+  } {
+    {{1 0 1}}
+    {{0 0 2} {1 0 4}} 
+  }
 }

 proc coltest {cmd} {
@@ -149,7 +153,7 @@ do_execsql_test 3.5.1 {
 #

 do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%);
  INSERT INTO t2 VALUES('a s h g s b j m r h', 's b p a d b b a o e');
  INSERT INTO t2 VALUES('r h n t a g r d d i', 'l d n j r c f t o q');
  INSERT INTO t2 VALUES('q k n i k c a a e m', 'c h n j p g s c i t');
@@ -218,24 +222,26 @@ proc rowidplus {cmd ival} {
 }
 sqlite3_fts5_create_function db rowidplus rowidplus

-do_execsql_test 4.2.1 {
-  INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) ');
-  SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
-} {
-  10 110
-}
-do_execsql_test 4.2.2 {
-  INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) ');
-  SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
-} {
-  10 121
-}
+if {[detail_is_full]} {
+  do_execsql_test 4.2.1 {
+    INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) ');
+    SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
+  } {
+    10 110
+  }
+  do_execsql_test 4.2.2 {
+    INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) ');
+    SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
+  } {
+    10 121
+  }

-do_execsql_test 4.2.3 {
-  SELECT rowid, rank FROM t2 
-  WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)'
-} {
-  10 122
+  do_execsql_test 4.2.3 {
+    SELECT rowid, rank FROM t2 
+      WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)'
+  } {
+    10 122
+  }
 }

 proc rowidmod {cmd imod} { 
@@ -243,7 +249,7 @@ proc rowidmod {cmd imod} {
 }
 sqlite3_fts5_create_function db rowidmod rowidmod
 do_execsql_test 4.3.1 {
-  CREATE VIRTUAL TABLE t3 USING fts5(x);
+  CREATE VIRTUAL TABLE t3 USING fts5(x, detail=%DETAIL%);
  INSERT INTO t3 VALUES('a one');
  INSERT INTO t3 VALUES('a two');
  INSERT INTO t3 VALUES('a three');
@@ -287,6 +293,7 @@ do_catchsql_test 4.4.4 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
 } {1 {parse error in rank function: }}

+} ;# foreach_detail_mode


 finish_test
@@ -22,7 +22,6 @@ ifcapable !fts5 {
  return
 }

-
 set data {
    -4026076
    {n x w k b p x b n t t d s}     {f j j s p j o}               
@@ -232,37 +231,9 @@ do_execsql_test 1.0 {

 fts5_aux_test_functions db

-proc matchdata {expr tbl collist {order ASC}} {
-
-  set cols ""
-  foreach e $collist {
-    append cols ", '$e'"
-  }
-
-  set tclexpr [db one [subst -novar {
-    SELECT fts5_expr_tcl(
-      $expr, 'nearset $cols -pc ::pc' [set cols]
-    )
-  }]]
-  set res [list]
-
-  db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
-    set cols [list]
-    foreach col $x(*) {
-      if {$col != "rowid"} { lappend cols $x($col) }
-    }
-    # set cols [list $a $b $c $d $e $f]
-    set ::pc 0
-    set rowdata [eval $tclexpr]
-    if {$rowdata != ""} { lappend res $x(rowid) $rowdata }
-  }
-
-  set res
-}
-
-proc do_auto_test {tn tbl cols expr} { 
+proc do_auto_test {tn tbl expr} {
  foreach order {asc desc} {
-    set res [matchdata $expr $tbl $cols $order]
+    set res [fts5_poslist_data $expr $tbl $order]
    set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]"

    set ::autotest_expr $expr
@@ -271,8 +242,6 @@ proc do_auto_test {tn tbl cols expr} {
      WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order]
    }] $res
  }
-
-
 }

 #-------------------------------------------------------------------------
@@ -332,7 +301,7 @@ for {set fold 0} {$fold < 3} {incr fold} {
    C.1 { a OR (b AND "b c") }
    C.2 { a OR (b AND "z c") }
  } {
-    do_auto_test 3.$fold.$tn tt {a b c d e f} $expr
+    do_auto_test 3.$fold.$tn tt $expr
  }
 }

@@ -366,11 +335,8 @@ foreach {tn expr} {
  4 {c1 : x} 5 {c2 : x} 6 {c3 : x}
  7 {c1 : y} 8 {c2 : y} 9 {c3 : y}
  10 {c1 : z} 11 {c2 : z} 12 {c3 : z}
-
-
 } {
-breakpoint
-  do_auto_test 4.$tn yy {c1 c2 c3} $expr
+  do_auto_test 4.$tn yy $expr
 }


@@ -0,0 +1,67 @@
+# 2016 Jan 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5bigtok
+
+proc rndterm {} {
+  set L [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
+  set l [lindex $L [expr int(rand() * [llength $L])]]
+  string repeat $l [expr int(rand() * 5) + 60]
+}
+
+proc rnddoc {n} {
+  set res [list]
+  for {set i 0} {$i < $n} {incr i} {
+    lappend res [rndterm]
+  }
+  set res
+}
+
+foreach_detail_mode $::testprefix {
+  db func rnddoc rnddoc
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+    CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, row);
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 )
+    INSERT INTO t1 SELECT rnddoc(3) FROM s;
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 )
+    INSERT INTO t1 SELECT rnddoc(3) FROM s;
+  }
+
+  foreach v [db eval {SELECT term FROM t1vocab}] {
+    set res [db eval {SELECT rowid FROM t1($v)}]
+    do_execsql_test 1.[string range $v 0 0] {
+      SELECT rowid FROM t1($v) ORDER BY rowid DESC
+    } [lsort -integer -decr $res]
+  }
+
+  do_execsql_test 2.0 {
+    INSERT INTO t1(t1) VALUES('optimize');
+  }
+
+  foreach v [db eval {SELECT term FROM t1vocab}] {
+    set res [db eval {SELECT rowid FROM t1($v)}]
+    do_execsql_test 2.[string range $v 0 0] {
+      SELECT rowid FROM t1($v) ORDER BY rowid DESC
+    } [lsort -integer -decr $res]
+  }
+}
+
+finish_test
+
+
@@ -42,6 +42,10 @@ foreach {tn opt} {
  1 {prefix=x}  
  2 {prefix='x'}
  3 {prefix='$'}
+  4 {prefix='1,2,'}
+  5 {prefix=',1'}
+  6 {prefix='1,2,3...'}
+  7 {prefix='1,2,3xyz'}
 } {
  set res [list 1 {malformed prefix=... directive}]
  do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
@@ -118,18 +122,15 @@ do_catchsql_test 5.3 {
 #-------------------------------------------------------------------------
 # Errors in prefix= directives.
 #
-do_catchsql_test 6.1 {
-  CREATE VIRTUAL TABLE abc USING fts5(a, prefix=1, prefix=2);
-} {1 {multiple prefix=... directives}}
 do_catchsql_test 6.2 {
  CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1, 2, 1001');
-} {1 {prefix length out of range: 1001}}
+} {1 {prefix length out of range (max 999)}}
 do_catchsql_test 6.3 {
  CREATE VIRTUAL TAbLE abc USING fts5(a, prefix='1, 2, 0000');
-} {1 {prefix length out of range: 0}}
+} {1 {prefix length out of range (max 999)}}
 do_catchsql_test 6.4 {
  CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1  , 1000000');
-} {1 {malformed prefix=... directive}}
+} {1 {prefix length out of range (max 999)}}

 #-------------------------------------------------------------------------
 # Duplicate tokenize= and other options.
@@ -160,6 +161,8 @@ do_catchsql_test 8.1 {
 #   9.1.* 'pgsz' options.
 #   9.2.* 'automerge' options.
 #   9.3.* 'crisismerge' options.
+#   9.4.* a non-existant option.
+#   9.5.* 'hashsize' options.
 #
 do_execsql_test 9.0 {
  CREATE VIRTUAL TABLE abc USING fts5(a, b);
@@ -204,5 +207,45 @@ do_catchsql_test 9.4.1 {
  INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
 } {1 {SQL logic error or missing database}}

+do_catchsql_test 9.5.1 {
+  INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
+} {1 {SQL logic error or missing database}}
+do_catchsql_test 9.5.2 {
+  INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
+} {1 {SQL logic error or missing database}}
+do_catchsql_test 9.5.3 {
+  INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
+} {0 {}}
+
+#-------------------------------------------------------------------------
+# Too many prefix indexes. Maximum allowed is 31.
+#
+foreach {tn spec} {
+  1 {prefix="1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
+  2 {prefix="1 2 3 4", prefix="5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
+} {
+  set sql "CREATE VIRTUAL TABLE xyz USING fts5(x, $spec)"
+  do_catchsql_test 10.$tn $sql {1 {too many prefix indexes (max 31)}}
+}
+
+#-------------------------------------------------------------------------
+# errors in the detail= option.
+#
+foreach {tn opt} {
+  1 {detail=x}  
+  2 {detail='x'}
+  3 {detail='$'}
+  4 {detail='1,2,'}
+  5 {detail=',1'}
+  6 {detail=''}
+} {
+  set res [list 1 {malformed detail=... directive}]
+  do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
+}
+
+do_catchsql_test 12.1 {
+  INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
+} {1 {SQL logic error or missing database}}
+
 finish_test

@@ -0,0 +1,70 @@
+# 2015 October 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5conflict
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, a, b);
+  CREATE VIRTUAL TABLE ft USING fts5(a, b, content=t1, content_rowid=x);
+}
+
+do_execsql_test 1.1 {
+  REPLACE INTO ft(rowid, a, b) VALUES(1, 'a b c', 'a b c');
+  REPLACE INTO t1              VALUES(1, 'a b c', 'a b c');
+}
+
+do_execsql_test 1.2 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.0 {
+  CREATE TABLE tbl(a INTEGER PRIMARY KEY, b, c);
+  CREATE VIRTUAL TABLE fts_idx USING fts5(b, c, content=tbl, content_rowid=a);
+  CREATE TRIGGER tbl_ai AFTER INSERT ON tbl BEGIN
+    INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c);
+  END;
+  CREATE TRIGGER tbl_ad AFTER DELETE ON tbl BEGIN
+    INSERT INTO fts_idx(fts_idx, rowid, b, c) 
+      VALUES('delete', old.a, old.b, old.c);
+  END;
+  CREATE TRIGGER tbl_au AFTER UPDATE ON tbl BEGIN
+    INSERT INTO fts_idx(fts_idx, rowid, b, c) 
+      VALUES('delete', old.a, old.b, old.c);
+    INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c);
+  END;
+}
+
+do_execsql_test 2.1 {
+  PRAGMA recursive_triggers = 1;
+  INSERT INTO tbl VALUES(1,   'x y z', '1 2 3');
+  INSERT INTO tbl VALUES(10,  'x y z', '1 2 3');
+  INSERT INTO tbl VALUES(100, 'x 1 z', '1 y 3');
+
+  UPDATE tbl SET b = '1 2 x' WHERE rowid=10;
+  REPLACE INTO tbl VALUES(1, '4 5 6', '3 2 1');
+  DELETE FROM tbl WHERE a=100;
+
+  INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
+}
+
+finish_test
+
+
@@ -250,8 +250,6 @@ foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] {
  }
 }

-}
-
 #------------------------------------------------------------------------
 #
 reset_db
@@ -335,6 +333,44 @@ do_catchsql_test 6.3.5 {
  INSERT INTO t1(t1) VALUES('integrity-check');
 } {1 {database disk image is malformed}}

+
+}
+
+#------------------------------------------------------------------------
+#
+reset_db
+reset_db
+proc rnddoc {n} {
+  set map [list a b c d]
+  set doc [list]
+  for {set i 0} {$i < $n} {incr i} {
+    lappend doc "x[lindex $map [expr int(rand()*4)]]"
+  }
+  set doc
+}
+
+db func rnddoc rnddoc
+do_test 7.0 {
+  execsql {
+    CREATE VIRTUAL TABLE t5 USING fts5(x);
+    INSERT INTO t5 VALUES( rnddoc(10000) );
+    INSERT INTO t5 VALUES( rnddoc(10000) );
+    INSERT INTO t5 VALUES( rnddoc(10000) );
+    INSERT INTO t5 VALUES( rnddoc(10000) );
+    INSERT INTO t5(t5) VALUES('optimize');
+  }
+} {}
+
+do_test 7.1 {
+  foreach i [db eval { SELECT rowid FROM t5_data WHERE rowid>100 }] {
+    db eval BEGIN  
+    db eval {DELETE FROM t5_data WHERE rowid = $i}
+    set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ]
+    if {$r != "1 {database disk image is malformed}"} { error $r }
+    db eval ROLLBACK  
+  }
+} {}
+
 sqlite3_fts5_may_be_corrupt 0
 finish_test

@@ -0,0 +1,244 @@
+# 2015 December 18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5detail
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+fts5_aux_test_functions db
+
+#--------------------------------------------------------------------------
+# Simple tests.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col);
+  INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
+  INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
+  INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
+  INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4
+  INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5
+  INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6
+  INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7
+  INSERT INTO t1 VALUES('c f j', 'j j i e a', 'h a c f d h e'); -- 8
+  INSERT INTO t1 VALUES('a h i', 'c i a f a', 'c f d h g d g'); -- 9
+  INSERT INTO t1 VALUES('j g g', 'e f e f f', 'h j b i c g e'); -- 10
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+foreach {tn match res} {
+  1 "a:a" {9}
+  2 "b:g" {1 4 6}
+  3 "c:h" {1 3 6 8 9 10}
+} {
+  do_execsql_test 1.2.$tn.1 {
+    SELECT rowid FROM t1($match);
+  } $res
+
+  do_execsql_test 1.2.$tn.2 {
+    SELECT rowid FROM t1($match || '*');
+  } $res
+}
+
+do_catchsql_test 1.3.1 {
+  SELECT rowid FROM t1('h + d');
+} {1 {fts5: phrase queries are not supported (detail!=full)}}
+
+do_catchsql_test 1.3.2 {
+  SELECT rowid FROM t1('NEAR(h d)');
+} {1 {fts5: NEAR queries are not supported (detail!=full)}}
+
+
+#-------------------------------------------------------------------------
+# integrity-check with both detail= and prefix= options.
+#
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1");
+  INSERT INTO t2(a) VALUES('aa ab');
+}
+
+#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r}
+
+do_execsql_test 2.1 {
+  INSERT INTO t2(t2) VALUES('integrity-check');
+}
+
+do_execsql_test 2.2 {
+  SELECT fts5_test_poslist(t2) FROM t2('aa');
+} {0.0.0}
+
+do_execsql_test 2.3 {
+  SELECT fts5_test_collist(t2) FROM t2('aa');
+} {0.0}
+
+set ::pc 0
+#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
+#exit
+
+#-------------------------------------------------------------------------
+# Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs
+# work with detail=col tables.
+#
+set data {
+  1  {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
+  2  {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc}
+  3  {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca}
+  4  {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca}
+  5  {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb}
+  6  {acc bba cba} {bab bbc bbb bcb aca} {bca ccc cbb aca bac ccc ccb}
+  7  {aba bab aaa} {abb bca aac bcb bcc} {bcb bbc aba aaa cba abc acc}
+  8  {cab aba aaa} {ccb aca caa bbc bcc} {aaa abc ccb bbb cac cca abb}
+  9  {bcb bab bac} {bcb cba cac bbb abc} {aba aca cbb acb abb ccc ccb}
+  10 {aba aab ccc} {abc ccc bcc cab bbb} {aab bcc cbb ccc aaa bac baa}
+  11 {bab acb cba} {aac cab cab bca cbc} {aab cbc aac baa ccb acc cac}
+  12 {ccc cbb cbc} {aaa aab bcc aac bbc} {cbc cbc bac bac ccc bbc acc}
+  13 {cab bbc abc} {bbb bab bba aca bab} {baa bbb aab bbb ccb bbb ccc}
+  14 {bbc cab caa} {acb aac abb cba acc} {cba bba bba acb abc abb baa}
+  15 {aba cca bcc} {aaa acb abc aab ccb} {cca bcb acc aaa caa cca cbc}
+  16 {bcb bba aba} {cbc acb cab caa ccb} {aac aaa bbc cab cca cba abc}
+  17 {caa cbb acc} {ccb bcb bca aaa bcc} {bbb aca bcb bca cbc cbc cca}
+  18 {cbb bbc aac} {ccc bbc aaa aab baa} {cab cab cac cca bbc abc bbc}
+  19 {ccc acc aaa} {aab cbb bca cca caa} {bcb aca aca cab acc bac bcc}
+  20 {aab ccc bcb} {bbc cbb bbc aaa bcc} {cbc aab ccc aaa bcb bac cbc}
+  21 {aba cab ccc} {bbc cbc cba acc bbb} {acc aab aac acb aca bca acb}
+  22 {bcb bca baa} {cca bbc aca ccb cbb} {aab abc bbc aaa cab bcc bcc}
+  23 {cac cbb caa} {bbc aba bbb bcc ccb} {bbc bbb cab bbc cac abb acc}
+  24 {ccb acb caa} {cab bba cac bbc aac} {aac bca abc cab bca cab bcb}
+  25 {bbb aca bca} {bcb acc ccc cac aca} {ccc acb acc cac cac bba bbc}
+  26 {bab acc caa} {caa cab cac bac aca} {aba cac caa acc bac ccc aaa}
+  27 {bca bca aaa} {ccb aca bca aaa baa} {bab acc aaa cca cba cca bac}
+  28 {ccb cac cac} {bca abb bba bbc baa} {aca ccb aac cab ccc cab caa}
+  29 {abc bca cab} {cac cbc cbb ccc bcc} {bcc aaa aaa acc aac cac aac}
+  30 {aca acc acb} {aab aac cbb caa acb} {acb bbc bbc acc cbb bbc aac}
+  31 {aba aca baa} {aca bcc cab bab acb} {bcc acb baa bcb bbc acc aba}
+  32 {abb cbc caa} {cba abb bbb cbb aca} {bac aca caa cac caa ccb bbc}
+  33 {bcc bcb bcb} {cca cab cbc abb bab} {caa bbc aac bbb cab cba aaa}
+  34 {caa cab acc} {ccc ccc bcc acb bcc} {bac bba aca bcb bba bcb cac}
+  35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb}
+  36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa}
+  37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb}
+  38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac}
+  39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba}
+  40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca}
+}
+
+set data {
+  1  {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
+}
+
+proc matchdata {expr {bAsc 1}} {
+
+  set tclexpr [db one {
+    SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z')
+  }]
+  set res [list]
+
+  #puts "$expr -> $tclexpr"
+  foreach {id x y z} $::data {
+    set cols [list $x $y $z]
+    set ::pc 0
+    #set hits [lsort -command instcompare [eval $tclexpr]]
+    set hits [eval $tclexpr]
+    if {[llength $hits]>0} {
+      lappend res [list $id $hits]
+    }
+  }
+
+  if {$bAsc} {
+    set res [lsort -integer -increasing -index 0 $res]
+  } else {
+    set res [lsort -integer -decreasing -index 0 $res]
+  }
+
+  return [concat {*}$res]
+}
+
+foreach {tn tbl} {
+  1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) }
+  2 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=none) }
+} {
+  reset_db
+  fts5_aux_test_functions db
+  execsql $tbl
+  foreach {id x y z} $data {
+    execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) }
+  }
+  foreach {tn2 expr} {
+    1 aaa    2 ccc    3 bab    4 aac
+    5 aa*    6 cc*    7 ba*    8 aa*
+    9 a*     10 b*   11 c*
+  } {
+
+    set res [matchdata $expr]
+
+    do_execsql_test 3.$tn.$tn2.1 {
+      SELECT rowid, fts5_test_poslist(t3) FROM t3($expr)
+    } $res
+
+    do_execsql_test 3.$tn.$tn2.2 {
+      SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr)
+    } $res
+  }
+}
+
+#-------------------------------------------------------------------------
+# Simple tests for detail=none tables.
+#
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none);
+  INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g');
+  INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9');
+}
+
+do_catchsql_test 4.1 {
+  SELECT * FROM t4('a:a')
+} {1 {fts5: column queries are not supported (detail=none)}}
+
+#-------------------------------------------------------------------------
+# Test that for the same content detail=none uses less space than 
+# detail=col, and that detail=col uses less space than detail=full
+#
+reset_db
+do_test 5.1 {
+  foreach {tbl detail} {t1 none t2 col t3 full} {
+    execsql "CREATE VIRTUAL TABLE $tbl USING fts5(x, y, z, detail=$detail)"
+    foreach {rowid x y z} $::data {
+      execsql "INSERT INTO $tbl (rowid, x, y, z) VALUES(\$rowid, \$x, \$y, \$z)"
+    }
+  }
+} {}
+
+do_execsql_test 5.2 {
+  SELECT 
+    (SELECT sum(length(block)) from t1_data) <
+    (SELECT sum(length(block)) from t2_data)
+} {1}
+
+do_execsql_test 5.3 {
+  SELECT 
+    (SELECT sum(length(block)) from t2_data) <
+    (SELECT sum(length(block)) from t3_data)
+} {1}
+
+
+
+finish_test
+
@@ -26,7 +26,7 @@ if { $tcl_platform(wordSize)<8 } {
  return
 }

-if 1 {
+foreach_detail_mode $testprefix {

 proc do_fb_test {tn sql res} {
  set res2 [lsort -integer -decr $res]
@@ -34,7 +34,7 @@ proc do_fb_test {tn sql res} {
  uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
 }

-# This test populates the FTS5 table containing $nEntry entries. Rows are 
+# This test populates the FTS5 table with $nEntry entries. Rows are 
 # numbered from 0 to ($nEntry-1). The rowid for row $i is:
 #
 #   ($iFirst + $i*$nStep)
@@ -77,10 +77,12 @@ proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} {
  do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc
  do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc
  
-  do_fb_test $tn.5.1 { 
-    SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
-  do_fb_test $tn.5.2 { 
-    SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc
+  if {[detail_is_full]} {
+    do_fb_test $tn.5.1 { 
+      SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
+    do_fb_test $tn.5.2 { 
+      SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc
+  }
 }


@@ -90,7 +92,7 @@ foreach {tn pgsz} {
 } {
  do_execsql_test $tn.0 { 
    DROP TABLE IF EXISTS t1;
-    CREATE VIRTUAL TABLE t1 USING fts5(x);
+    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
    INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
  }

@@ -107,7 +109,7 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} {
  execsql {
    BEGIN;
    DROP TABLE IF EXISTS t1;
-    CREATE VIRTUAL TABLE t1 USING fts5(x);
+    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
    INSERT INTO t1 VALUES('b a');

@@ -130,8 +132,6 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} {

 do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]

-}
-
 #--------------------------------------------------------------------
 #
 reset_db
@@ -158,7 +158,7 @@ proc rnddoc {} {
 db func rnddoc rnddoc

 do_execsql_test 3.1 {
-  CREATE VIRTUAL TABLE abc USING fts5(a);
+  CREATE VIRTUAL TABLE abc USING fts5(a, detail=%DETAIL%);
  INSERT INTO abc(abc, rank) VALUES('pgsz', 32);

  INSERT INTO abc VALUES ( rnddoc() );
@@ -192,6 +192,9 @@ foreach v $vocab {
  } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
 }

+} ;# foreach_detail_mode
+
+

 finish_test

@@ -42,6 +42,15 @@ foreach {tn expr res} {
  10 {abc + "" + def}                 {"abc" + "def"}
  11 {abc "" def}                     {"abc" AND "def"}
  12 {r+e OR w}                       {"r" + "e" OR "w"}
+
+  13 {a AND b NOT c}                  {"a" AND ("b" NOT "c")}
+  14 {a OR b NOT c}                   {"a" OR ("b" NOT "c")}
+  15 {a NOT b AND c}                  {("a" NOT "b") AND "c"}
+  16 {a NOT b OR c}                   {("a" NOT "b") OR "c"}
+
+  17 {a AND b OR c}                   {("a" AND "b") OR "c"}
+  18 {a OR b AND c}                   {"a" OR ("b" AND "c")}
+
 } {
  do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
 }
@@ -52,7 +52,7 @@ do_faultsim_test 2 -prep {
    INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
  }
 } -test {
-  faultsim_test_result {0 {}} 
+  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
 }

 reset_db
@@ -66,13 +66,13 @@ do_faultsim_test 3 -prep {
 } -body {
  execsql { DELETE FROM t1 }
 } -test {
-  faultsim_test_result {0 {}} 
+  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
 }

 reset_db
 do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b);
-  INSERT INTO t2 VALUES('m f a jj th q jr ar',   'hj n h h sg j i m');
+  INSERT INTO t2 VALUES('m f a jj th q gi ar',   'hj n h h sg j i m');
  INSERT INTO t2 VALUES('nr s t g od j kf h',    'sb h aq rg op rb n nl');
  INSERT INTO t2 VALUES('do h h pb p p q fr',    'c rj qs or cr a l i');
  INSERT INTO t2 VALUES('lk gp t i lq mq qm p',  'h mr g f op ld aj h');
@@ -95,13 +95,14 @@ foreach {tn expr res} {
  7 { NEAR(r a, 5) }  {9}
  8 { m* f* }         {1 4 6 8 9 10}
  9 { m* + f* }       {1 8}
+  10 { c NOT p }       {5 6 7 10}
 } {
  do_faultsim_test 4.$tn -prep {
    faultsim_restore_and_reopen
  } -body "
    execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
  " -test "
-    faultsim_test_result {[list 0 $res]}
+    faultsim_test_result {[list 0 $res]} {1 {vtable constructor failed: t2}}
  "
 }

@@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5fault2

-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -40,27 +40,6 @@ do_faultsim_test 1 -faults oom-* -prep {
  faultsim_test_result [list 0 {}]
 }

-#-------------------------------------------------------------------------
-# An OOM within an "ORDER BY rank" query.
-#
-db func rnddoc fts5_rnddoc 
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE xx USING fts5(x);
-  INSERT INTO xx VALUES ('abc ' || rnddoc(10));
-  INSERT INTO xx VALUES ('abc abc' || rnddoc(9));
-  INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8));
-} {}
-faultsim_save_and_close
-
-do_faultsim_test 2 -faults oom-* -prep {
-  faultsim_restore_and_reopen
-  execsql { SELECT * FROM xx }
-} -body {
-  execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank }
-} -test {
-  faultsim_test_result [list 0 {3 2 1}]
-}
-
 #-------------------------------------------------------------------------
 # An OOM while "reseeking" an FTS cursor.
 #
@@ -107,7 +86,7 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]
 do_faultsim_test 4 -faults oom-* -body {
  db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
 } -test {
-  faultsim_test_result {0 {0 {} 3}}
+  faultsim_test_result {0 {0 {} 4}}
 }

 #-------------------------------------------------------------------------
@@ -65,19 +65,26 @@ do_faultsim_test 2.2 -faults oom-t* -body {
 }

 #-------------------------------------------------------------------------
-# OOM while scanning an fts5vocab table.
+# OOM while scanning fts5vocab tables.
 #
 reset_db
 do_test 3.0 {
  execsql {
    CREATE VIRTUAL TABLE tt USING fts5(x);
    CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row');
+
+    CREATE VIRTUAL TABLE tt2 USING fts5(x, detail=col);
+    CREATE VIRTUAL TABLE tv2 USING fts5vocab(tt2, 'col');
+
    INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
+    INSERT INTO tt2(tt2, rank) VALUES('pgsz', 32);
    BEGIN;
  }
+
  for {set i 0} {$i < 20} {incr i} {
    set str [string repeat "$i " 50]
    execsql { INSERT INTO tt VALUES($str) }
+    execsql { INSERT INTO tt2 VALUES($str) }
  }
  execsql COMMIT
 } {}
@@ -90,6 +97,36 @@ do_faultsim_test 3.1 -faults oom-t* -body {
  faultsim_test_result {0 {0 1 10 11 12 13 14 15 16 17 18 19 2 3 4 5 6 7 8 9}}
 }

+do_faultsim_test 3.2 -faults oom-t* -body {
+  db eval {
+    SELECT term FROM tv WHERE term BETWEEN '1' AND '2';
+  }
+} -test {
+  faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
+}
+
+breakpoint
+do_execsql_test 3.3.0 {
+  SELECT * FROM tv2;
+} {
+  0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        
+  14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     
+  2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   
+  9 x 1 {}
+}
+do_faultsim_test 3.3 -faults oom-t* -body {
+  db eval {
+    SELECT * FROM tv2;
+  }
+} -test {
+  faultsim_test_result [list 0 [list                                   \
+      0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        \
+      14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     \
+      2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   \
+      9 x 1 {}
+  ]]
+}
+


 finish_test
@@ -87,5 +87,33 @@ do_faultsim_test 1 -faults oom-t* -prep {
  faultsim_test_result {0 {}}
 }

+#-------------------------------------------------------------------------
+# Test fault-injection when a segment is promoted.
+#
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE xy USING fts5(x);
+  INSERT INTO xy(rowid, x) VALUES(1, '1 2 3');
+  INSERT INTO xy(rowid, x) VALUES(2, '2 3 4');
+  INSERT INTO xy(rowid, x) VALUES(3, '3 4 5');
+}
+faultsim_save_and_close
+
+do_faultsim_test 2.1 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  db eval { UPDATE OR REPLACE xy SET rowid=3 WHERE rowid = 2 }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+# Test fault-injection when an empty expression is parsed.
+#
+do_faultsim_test 2.2 -faults oom-* -body {
+  db eval { SELECT * FROM xy('""') }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
 finish_test

@@ -0,0 +1,60 @@
+# 2015 September 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5fault8
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+  fts5_aux_test_functions db
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+    INSERT INTO t1 VALUES('a b c d', '1 2 3 4');
+    INSERT INTO t1 VALUES('a b a b', NULL);
+    INSERT INTO t1 VALUES(NULL, '1 2 1 2');
+  }
+  
+  do_faultsim_test 1 -faults oom-* -body {
+    execsql { 
+      SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' 
+    }
+  } -test {
+    faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \
+                         {1 SQLITE_NOMEM}
+  }
+  
+  do_faultsim_test 2 -faults oom-* -body {
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+  } -test {
+    faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
+  }
+
+  if {[detail_is_none]==0} {
+    do_faultsim_test 3 -faults oom-* -body {
+      execsql { SELECT rowid FROM t1('b:2') }
+    } -test {
+      faultsim_test_result {0 {1 3}} {1 SQLITE_NOMEM}
+    }
+  }
+} ;# foreach_detail_mode...
+
+finish_test
+
@@ -0,0 +1,156 @@
+# 2015 September 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5fault9
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+fts5_aux_test_functions db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  WITH seq(s) AS ( SELECT 1 UNION ALL SELECT s+1 FROM seq WHERE s<50)
+  INSERT INTO t1 SELECT 'x x x y y y', 'a b c d e f' FROM seq;
+}
+
+do_faultsim_test 1 -faults oom-* -body {
+  execsql { SELECT count(*) FROM t1('x AND y') }
+} -test {
+  faultsim_test_result {0 50}
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%);
+  INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
+  INSERT INTO t2 VALUES('abc cba', 'cba abc');
+  INSERT INTO t2 VALUES('abc cba', 'cba abc');
+  INSERT INTO t2 VALUES('abc cba', 'cba abc');
+
+  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+}
+
+do_faultsim_test 2 -faults oom-* -body {
+  execsql { SELECT count(*) FROM t2('a* AND c*') }
+} -test {
+  faultsim_test_result {0 6}
+}
+
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t3 USING fts5(a, detail=%DETAIL%);
+  INSERT INTO t3 VALUES('a x x a x a a a');
+  INSERT INTO t3 VALUES('x a a x a x x x');
+}
+
+do_faultsim_test 3.1 -faults oom-* -body {
+  execsql { SELECT highlight(t3, 0, '[', ']') FROM t3('a') }
+} -test {
+  faultsim_test_result {0 {{[a] x x [a] x [a] [a] [a]} {x [a] [a] x [a] x x x}}}
+}
+
+do_faultsim_test 3.2 -faults oom-t* -body {
+  execsql { SELECT fts5_test_poslist2(t3) FROM t3('x') }
+} -test {
+  faultsim_test_result \
+      {0 {{0.0.1 0.0.2 0.0.4} {0.0.0 0.0.3 0.0.5 0.0.6 0.0.7}}} \
+      {1 SQLITE_NOMEM}
+}
+
+#-------------------------------------------------------------------------
+# Test OOM injection with the xPhraseFirstColumn() API and a tokenizer
+# uses query synonyms.
+#
+fts5_tclnum_register db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t4 USING fts5(x, y, z, detail=%DETAIL%, tokenize=tclnum);
+  INSERT INTO t4 VALUES('one two three', '1 2 3', 'i ii iii');
+  INSERT INTO t4 VALUES('1 2 3', 'i ii iii', 'one two three');
+  INSERT INTO t4 VALUES('i ii iii', 'one two three', 'i ii iii');
+
+  INSERT INTO t4 VALUES('a1 a2 a3', 'a4 a5 a6', 'a7 a8 a9');
+  INSERT INTO t4 VALUES('b1 b2 b3', 'b4 b5 b6', 'b7 b8 b9');
+  INSERT INTO t4 VALUES('c1 c2 c3', 'c4 c5 c6', 'c7 c8 c9');
+}
+
+do_faultsim_test 4.1 -faults oom-t* -body {
+  execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
+} -test {
+  faultsim_test_result \
+      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
+}
+
+do_faultsim_test 4.2 -faults oom-t* -body {
+  execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
+} -test {
+  faultsim_test_result \
+      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
+}
+
+
+#-------------------------------------------------------------------------
+# An OOM within an "ORDER BY rank" query.
+#
+db func rnddoc fts5_rnddoc 
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=%DETAIL%);
+  INSERT INTO xx VALUES ('def', 'abc ' || rnddoc(10));
+  INSERT INTO xx VALUES ('def', 'abc abc' || rnddoc(9));
+  INSERT INTO xx VALUES ('def', 'abc abc abc' || rnddoc(8));
+} {}
+faultsim_save_and_close
+
+do_faultsim_test 5 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT * FROM xx }
+} -body {
+  execsql { SELECT rowid FROM xx('abc AND def') ORDER BY rank }
+} -test {
+  faultsim_test_result [list 0 {3 2 1}]
+}
+
+set doc [string repeat "xyz " 500]
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE yy USING fts5(y, detail=%DETAIL%);
+  INSERT INTO yy(yy, rank) VALUES('pgsz', 64);
+  INSERT INTO yy VALUES ($doc);
+  INSERT INTO yy VALUES ('1 2 3');
+  INSERT INTO yy VALUES ('xyz');
+  UPDATE yy SET y = y WHERE rowid = 1;
+  UPDATE yy SET y = y WHERE rowid = 1;
+  UPDATE yy SET y = y WHERE rowid = 1;
+  UPDATE yy SET y = y WHERE rowid = 1;
+} {}
+
+do_faultsim_test 6 -faults oom-* -body {
+  execsql { SELECT rowid FROM yy('xyz') }
+} -test {
+  faultsim_test_result [list 0 {1 3}]
+}
+
+
+} ;# foreach_detail_mode...
+
+finish_test
+
@@ -0,0 +1,64 @@
+# 2016 February 2
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultA
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE o1 USING fts5(a, detail=%DETAIL%);
+    INSERT INTO o1(o1, rank) VALUES('pgsz', 32);
+  
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 )
+    INSERT INTO o1 SELECT 'A B C' FROM s;
+  
+    INSERT INTO o1 VALUES('A X C');
+  
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 )
+    INSERT INTO o1 SELECT 'A B C' FROM s;
+  }
+  
+  do_faultsim_test 1 -faults oom* -prep {
+    sqlite3 db test.db
+  } -body {
+    execsql { SELECT rowid FROM o1('a NOT b') }
+  } -test {
+    faultsim_test_result {0 301}
+  }
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE o2 USING fts5(a);
+  
+  INSERT INTO o2 VALUES('A B C');
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 )
+  INSERT INTO o2 SELECT group_concat('A B C ') FROM s;
+}
+
+do_faultsim_test 2 -faults oom* -prep {
+  sqlite3 db test.db
+} -body {
+  execsql { SELECT rowid FROM o2('a+b+c NOT xyz') }
+} -test {
+  faultsim_test_result {0 {1 2}}
+}
+finish_test
+
@@ -64,11 +64,13 @@ proc random_doc {vocab nWord} {
  return $doc
 }

+foreach_detail_mode $testprefix {
+
 set vocab [build_vocab1]
 db func r random_doc 

 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE eee USING fts5(e, ee);
+  CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
  BEGIN;
    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
    INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
@@ -84,12 +86,14 @@ lappend vocab xyz
 do_execsql_test 1.1 {
  CREATE VIRTUAL TABLE vocab USING fts5vocab(eee, 'row'); 
  BEGIN;
-    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
-    INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
-    INSERT INTO eee(eee) VALUES('integrity-check');
 }
-
 do_test 1.2 {
+  for {set i 1} {$i <= 100} {incr i} {
+    execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) }
+  }
+} {}
+  
+do_test 1.3 {
  db eval { SELECT term, doc FROM vocab } {
    set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
    if {$nRow != $doc} {
@@ -99,10 +103,12 @@ do_test 1.2 {
  set {} {}
 } {}

-do_execsql_test 1.3 {
+do_execsql_test 1.4 {
  COMMIT;
  INSERT INTO eee(eee) VALUES('integrity-check');
 }

+} ;# foreach_detail_mode
+
 finish_test

@@ -9,7 +9,7 @@
 #
 #***********************************************************************
 #
-# This file containst tests focused on the integrity-check procedure.
+# This file contains tests focused on the integrity-check procedure.
 #

 source [file join [file dirname [info script]] fts5_common.tcl]
@@ -102,6 +102,112 @@ do_catchsql_test 4.5 {
 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM zz_data} {puts $r}
 #exit

+execsql { ROLLBACK }
+
+
+#-------------------------------------------------------------------------
+# Test that integrity-check works on a reasonably large db with many
+# different terms.
+
+# Document generator command.
+proc rnddoc {n} {
+  set doc [list]
+  for {set i 0} {$i<$n} {incr i} {
+    lappend doc [format %.5d [expr int(rand()*10000)]]
+  }
+  return $doc
+}
+db func rnddoc rnddoc
+
+expr srand(0)
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE gg USING fts5(a, prefix="1,2,3");
+  INSERT INTO gg(gg, rank) VALUES('pgsz', 256);
+  INSERT INTO gg VALUES(rnddoc(20));
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+}
+
+do_execsql_test 5.1 {
+  INSERT INTO gg(gg) VALUES('integrity-check');
+}
+
+do_execsql_test 5.2 {
+  INSERT INTO gg(gg) VALUES('optimize');
+}
+
+do_execsql_test 5.3 {
+  INSERT INTO gg(gg) VALUES('integrity-check');
+}
+
+do_test 5.4.1 {
+  set ok 0
+  for {set i 0} {$i < 10000} {incr i} {
+    set T [format %.5d $i]
+    set res  [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC  }]
+    set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }]
+    if {$res == [lsort -integer $res2]} { incr ok }
+  }
+  set ok
+} {10000}
+
+do_test 5.4.2 {
+  set ok 0
+  for {set i 0} {$i < 100} {incr i} {
+    set T "[format %.3d $i]*"
+    set res  [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC  }]
+    set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }]
+    if {$res == [lsort -integer $res2]} { incr ok }
+  }
+  set ok
+} {100}
+
+#-------------------------------------------------------------------------
+# Similar to 5.*.
+#
+foreach {tn pgsz} {
+  1  32
+  2  36
+  3  40
+  4  44
+  5  48
+} {
+  do_execsql_test 6.$tn.1 {
+    DROP TABLE IF EXISTS hh;
+    CREATE VIRTUAL TABLE hh USING fts5(y);
+    INSERT INTO hh(hh, rank) VALUES('pgsz', $pgsz);
+
+    WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999)
+     INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1)
+     FROM s;
+
+    WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999)
+     INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1)
+     FROM s;
+
+    INSERT INTO hh(hh) VALUES('optimize');
+  }
+
+  do_test 6.$tn.2 {
+    set ok 0
+    for {set i 0} {$i < 1000} {incr i} {
+      set T [format %.3d%.3d%.3d $i $i $i]
+      set res  [db eval { SELECT rowid FROM hh($T) ORDER BY rowid ASC  }]
+      set res2 [db eval { SELECT rowid FROM hh($T) ORDER BY rowid DESC }]
+      if {$res == [lsort -integer $res2]} { incr ok }
+    }
+    set ok
+  } {1000}
+}

 finish_test

@@ -16,6 +16,8 @@ set testprefix fts5matchinfo
 # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 { finish_test ; return }

+foreach_detail_mode $testprefix {
+
 proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
@@ -27,7 +29,7 @@ db func mit mit
 sqlite3_fts5_register_matchinfo db

 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(content);
+  CREATE VIRTUAL TABLE t1 USING fts5(content, detail=%DETAIL%);
 } 

 do_execsql_test 1.1 {
@@ -41,7 +43,7 @@ do_execsql_test 1.1 {
 # Now create an FTS4 table that does not specify matchinfo=fts3.
 #
 do_execsql_test 1.2 {
-  CREATE VIRTUAL TABLE t2 USING fts5(content);
+  CREATE VIRTUAL TABLE t2 USING fts5(content, detail=%DETAIL%);
  INSERT INTO t2 SELECT * FROM t1;
  SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';
 } {{1 1 1 2 2} {1 1 1 2 2}}
@@ -149,9 +151,17 @@ proc normalize2 {list_of_lists} {
  return $res
 }

+# Similar to [do_matchinfo_test], except that this is a no-op if the FTS5
+# mode is not detail=full.
+#
+proc do_matchinfo_p_test {tn tbl expr results} {
+  if {[detail_is_full]} {
+    uplevel [list do_matchinfo_test $tn $tbl $expr $results]
+  }
+}

 do_execsql_test 4.1.0 {
-  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
+  CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%);
  INSERT INTO t4 VALUES('a b c d e', 'f g h i j');
  INSERT INTO t4 VALUES('f g h i j', 'a b c d e');
 }
@@ -185,7 +195,7 @@ do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
  xpxsscplax -
 }

-do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
+do_matchinfo_p_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
  p {1 1}
  c {2 2}
  x {
@@ -203,8 +213,8 @@ do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
 }

 do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'}     { s {{2 0} {0 2}} }
-do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
-do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
+do_matchinfo_p_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
+do_matchinfo_p_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
 do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'}     { s {{1 0} {0 1}} }
 do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} {
  x { 
@@ -220,7 +230,7 @@ do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f NOT abcd'} {
 }

 do_execsql_test 4.2.0 {
-  CREATE VIRTUAL TABLE t5 USING fts5(content);
+  CREATE VIRTUAL TABLE t5 USING fts5(content, detail=%DETAIL%);
  INSERT INTO t5 VALUES('a a a a a');
  INSERT INTO t5 VALUES('a b a b a');
  INSERT INTO t5 VALUES('c b c b c');
@@ -233,7 +243,7 @@ do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'}         {
 do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'}         { s {2} }
 do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'}       { s {3} }
 do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
-do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
+do_matchinfo_p_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
 do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1} }

 do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";
@@ -250,7 +260,7 @@ if 0 {
 do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'}         { s {2} }
 do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'}       { s {3} }
 do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
-do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
+do_matchinfo_p_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
 do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1 1} }

 do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') }
@@ -260,10 +270,10 @@ do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'}         { s {2 1} }
 do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
 do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'}       { s {3} }
 do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
-do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
+do_matchinfo_p_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }

 do_execsql_test 4.5.0 {
-  CREATE VIRTUAL TABLE t6 USING fts5(a, b, c);
+  CREATE VIRTUAL TABLE t6 USING fts5(a, b, c, detail=%DETAIL%);
  INSERT INTO t6 VALUES('a', 'b', 'c');
 }
 do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'}       { s {{1 1 1}} }
@@ -274,7 +284,7 @@ do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'}       { s {{1 1 1}} }
 # use the full-text index (i.e. lookup by rowid or full-table scan).
 #
 do_execsql_test 7.1 {
-  CREATE VIRTUAL TABLE t10 USING fts5(content);
+  CREATE VIRTUAL TABLE t10 USING fts5(content, detail=%DETAIL%);
  INSERT INTO t10 VALUES('first record');
  INSERT INTO t10 VALUES('second record');
 }
@@ -299,7 +309,7 @@ do_execsql_test 7.4 {
 #   SELECT sum(length(content)) < count(*) FROM fts4table;
 #
 do_execsql_test 8.1 {
-  CREATE VIRTUAL TABLE t11 USING fts5(content);
+  CREATE VIRTUAL TABLE t11 USING fts5(content, detail=%DETAIL%);
  INSERT INTO t11(t11, rank) VALUES('pgsz', 32);
  INSERT INTO t11 VALUES('quitealongstringoftext');
  INSERT INTO t11 VALUES('anotherquitealongstringoftext');
@@ -318,22 +328,24 @@ do_execsql_test 8.3 {

 #-------------------------------------------------------------------------

-do_execsql_test 9.1 {
-  CREATE VIRTUAL TABLE t12 USING fts5(content);
-  INSERT INTO t12 VALUES('a b c d');
-  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
-} {{0 1 1 0 1 1 1 1 1}}
-do_execsql_test 9.2 {
-  INSERT INTO t12 VALUES('a d c d');
-  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
-} {
-  {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2}
-}
-do_execsql_test 9.3 {
-  INSERT INTO t12 VALUES('a d d a');
-  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
-} {
-  {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3}
+if {[detail_is_full]} {
+  do_execsql_test 9.1 {
+    CREATE VIRTUAL TABLE t12 USING fts5(content, detail=%DETAIL%);
+    INSERT INTO t12 VALUES('a b c d');
+    SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
+  } {{0 1 1 0 1 1 1 1 1}}
+  do_execsql_test 9.2 {
+    INSERT INTO t12 VALUES('a d c d');
+    SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
+  } {
+    {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2}
+  }
+  do_execsql_test 9.3 {
+    INSERT INTO t12 VALUES('a d d a');
+    SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
+  } {
+    {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3}
+  }
 }

 #---------------------------------------------------------------------------
@@ -341,7 +353,7 @@ do_execsql_test 9.3 {
 #
 do_execsql_test 10.1 {
  DROP TABLE t10;
-  CREATE VIRTUAL TABLE t10 USING fts5(idx, value);
+  CREATE VIRTUAL TABLE t10 USING fts5(idx, value, detail=%DETAIL%);
  INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three');
  SELECT t10.rowid, t10.*
    FROM t10
@@ -358,7 +370,7 @@ do_execsql_test 10.1 {
 reset_db
 sqlite3_fts5_register_matchinfo db
 do_execsql_test 11.0 {
-  CREATE VIRTUAL TABLE tt USING fts5(x, y);
+  CREATE VIRTUAL TABLE tt USING fts5(x, y, detail=%DETAIL%);
  INSERT INTO tt VALUES('c d a c d d', 'e a g b d a');   -- 1
  INSERT INTO tt VALUES('c c g a e b', 'c g d g e c');   -- 2
  INSERT INTO tt VALUES('b e f d e g', 'b a c b c g');   -- 3
@@ -410,6 +422,8 @@ foreach {tn expr res} {
  }

 } {
+
+  if {[string match *:* $expr] && [detail_is_none]} continue
  do_execsql_test 11.1.$tn.1  {
    SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr
  } $res
@@ -443,7 +457,7 @@ db func mit mit
 do_test 12.0 {
  set cols [list]
  for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" }
-  execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])"
+  execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,], detail=%DETAIL%)"
 } {}

 do_execsql_test 12.1 {
@@ -451,5 +465,7 @@ do_execsql_test 12.1 {
  SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';
 } [list [list [expr 1<<4] [expr 1<<(45-32)]]]

+} ;# foreach_detail_mode
+
 finish_test

@@ -0,0 +1,59 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test that focus on incremental merges of segments.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5merge
+
+proc dump_structure {} {
+  db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
+    foreach lvl [lrange $t 1 end] {
+      set seg [string repeat . [expr [llength $lvl]-2]]
+      puts "[lrange $lvl 0 1] $seg"
+    }
+  }
+}
+
+foreach_detail_mode $testprefix {
+
+if {[detail_is_none]==0} continue
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  INSERT INTO t1(t1, rank) VALUES('crisismerge', 2);
+  INSERT INTO t1 VALUES('1 2 3 4');
+}
+
+expr srand(0)
+db func rnddoc fts5_rnddoc
+do_test 1.1 {
+  for {set i 0} {$i < 100} {incr i} {
+    execsql {
+      BEGIN;
+        DELETE FROM t1 WHERE rowid = 1;
+        INSERT INTO t1(rowid, x) VALUES(1, '1 2 3 4');
+        INSERT INTO t1 VALUES(rnddoc(10));
+      COMMIT;
+    }
+  }
+} {}
+
+do_execsql_test 1.2 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+}
+
+finish_test
+
@@ -195,6 +195,150 @@ for {set x 0} {$x<2} {incr x} {
  execsql { INSERT INTO t3(t3) VALUES('integrity-check') }
 }

+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(c1, c2);
+  INSERT INTO t2 VALUES('xa xb', 'xb xa');
+
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 2
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 4
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 8
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 16
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 32
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 64
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 128
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 256
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 512
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 1024
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 2048
+  INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 4096
+
+  SELECT count(*) FROM t2('x*');
+} {4096}
+
+do_execsql_test 4.1 {
+  UPDATE t2 SET c2 = 'ya yb';
+  SELECT count(*) FROM t2('c1:x*');
+  SELECT count(*) FROM t2('c2:x*');
+} {4096 0}
+
+do_execsql_test 4.2 {
+  UPDATE t2 SET c2 = 'xa';
+  SELECT count(*) FROM t2('c1:x*');
+  SELECT count(*) FROM t2('c2:x*');
+} {4096 4096}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+proc rnddoc {n} {
+  set map [list a b c d]
+  set doc [list]
+  for {set i 0} {$i < $n} {incr i} {
+    lappend doc "x[lindex $map [expr int(rand()*4)]]"
+  }
+  set doc
+}
+set cols [list]
+for {set i 1} {$i<250} {incr i} { 
+  lappend cols "c$i" 
+  lappend vals "'[rnddoc 10]'"
+}
+
+do_test 5.0 {
+  execsql "CREATE VIRTUAL TABLE t4 USING fts5([join $cols ,])"
+  execsql {INSERT INTO t4(t4, rank) VALUES('pgsz', 32)}
+  execsql "INSERT INTO t4 VALUES([join $vals ,])"
+  execsql "INSERT INTO t4 VALUES([join $vals ,])"
+  execsql "INSERT INTO t4 VALUES([join $vals ,])"
+  execsql "INSERT INTO t4 VALUES([join $vals ,])"
+} {}
+
+proc gmatch {col pattern} {
+  expr {[lsearch -glob $col $pattern]>=0}
+}
+db func gmatch gmatch
+foreach {tn col pattern} {
+  1 c100 {xa*}
+  2 c200 {xb*}
+} {
+  set res [db eval "SELECT rowid FROM t4 WHERE gmatch($col, \$pattern)"]
+  set query "$col : $pattern"
+  do_execsql_test 5.$tn { SELECT rowid FROM t4($query) } $res
+}
+
+reset_db
+db func fts5_rnddoc fts5_rnddoc
+do_test 6.0 {
+  execsql {
+    CREATE VIRTUAL TABLE t5 USING fts5(x, y);
+    INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) );
+    INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) );
+    INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) );
+    INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) );
+  }
+} {}
+
+proc gmatch {col pattern} {
+  expr {[lsearch -glob $col $pattern]>=0}
+}
+db func gmatch gmatch
+foreach {tn col pattern} {
+  1 y {xa*}
+  2 y {xb*}
+  3 y {xc*}
+  4 x {xa*}
+  5 x {xb*}
+  6 x {xc*}
+} {
+  set res [db eval "SELECT rowid FROM t5 WHERE gmatch($col, \$pattern)"]
+  set query "$col : $pattern"
+  do_execsql_test 6.$tn { SELECT rowid FROM t5($query) } $res
+}
+
+#-------------------------------------------------------------------------
+# Check that the various ways of creating prefix indexes produce the
+# same database on disk.
+#
+save_prng_state
+foreach {tn create} {
+  1 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1,2,3") }
+  2 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1 2 3") }
+  3 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix=1, prefix=2, prefix=3) }
+  4 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1 2", prefix=3) }
+} {
+  execsql { DROP TABLE IF EXISTS tt }
+  restore_prng_state
+  execsql $create
+  execsql {
+    INSERT INTO tt VALUES('cc b ggg ccc aa eee hh', 'aa g b hh a e');
+    INSERT INTO tt VALUES('cc bb cc gg j g cc', 'ii jjj ggg jjj cc cc');
+    INSERT INTO tt VALUES('h eee cc h iii', 'aaa iii dd iii dd');
+    INSERT INTO tt VALUES('jjj hh eee c e b gg', 'j bbb jj ddd jj');
+    INSERT INTO tt VALUES('ii hhh aaa ff c hhh iii', 'j cc hh bb e');
+    INSERT INTO tt VALUES('e fff hhh i aaa', 'g b aa gg c aa dd');
+    INSERT INTO tt VALUES('i aaa ccc gg hhh aa h', 'j bbb bbb d ff');
+    INSERT INTO tt VALUES('g f gg ff ff jjj d', 'jjj d j fff fff ee j');
+    INSERT INTO tt VALUES('a cc e ccc jjj c', 'ccc iii d bb a eee g');
+    INSERT INTO tt VALUES('jj hh hh bb bbb gg', 'j c jjj bb iii f');
+    INSERT INTO tt VALUES('a ggg g cc ccc aa', 'jjj j j aaa c');
+    INSERT INTO tt VALUES('ddd j dd b i', 'aaa bbb iii ggg ff ccc ddd');
+    INSERT INTO tt VALUES('jj ii hh c ii h gg', 'hhh bbb ddd bbb hh g ggg');
+    INSERT INTO tt VALUES('aa hhh ccc h ggg ccc', 'iii d jj a ff ii');
+  }
+
+  #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM tt_data} {puts $r}
+
+  if {$tn==1} {
+    set ::checksum [execsql {SELECT md5sum(id, block) FROM tt_data}]
+  } else {
+    do_execsql_test 7.$tn {
+      SELECT md5sum(id, block) FROM tt_data
+    } [list $::checksum]
+  }
+}

 finish_test

@@ -0,0 +1,83 @@
+# 2015 October 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5query
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
+  reset_db
+  do_test 1.$tn.1 {
+    execsql {
+      CREATE VIRTUAL TABLE t1 USING fts5(x);
+      INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
+      BEGIN;
+    }
+    foreach x [list aaa bbb ccc ddd eee fff ggg hhh iii jjj] {
+      set doc [string repeat "$x " 30]
+      execsql { INSERT INTO t1 VALUES($doc) }
+    }
+    execsql COMMIT
+  } {}
+
+  do_execsql_test 1.$tn.2 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+
+  set ret 1
+  foreach x [list a b c d e f g h i j] {
+    do_execsql_test 1.$tn.3.$ret {
+      SELECT rowid FROM t1 WHERE t1 MATCH $x || '*';
+    } $ret
+    incr ret
+  }
+}
+
+for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
+  reset_db
+  do_test 2.$tn.1 {
+    execsql {
+      CREATE VIRTUAL TABLE t1 USING fts5(x);
+      INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
+      BEGIN;
+    }
+    foreach x [list bbb ddd fff hhh jjj lll nnn ppp rrr ttt] {
+      set doc [string repeat "$x " 30]
+      execsql { INSERT INTO t1 VALUES($doc) }
+    }
+    execsql COMMIT
+  } {}
+
+  do_execsql_test 1.$tn.2 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+
+  set ret 1
+  foreach x [list a c e g i k m o q s u] {
+    do_execsql_test 2.$tn.3.$ret {
+      SELECT rowid FROM t1 WHERE t1 MATCH $x || '*';
+    } {}
+    incr ret
+  }
+}
+
+
+finish_test
+
+
@@ -41,5 +41,60 @@ do_execsql_test 1.3 {
  WHERE xyz MATCH 'x AND y' ORDER BY rank
 } [list [string map {x [x] y [y]} $doc]]

+#-------------------------------------------------------------------------
+# Check that the 'rank' option really is persistent.
+#
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE tt USING fts5(a);
+  INSERT INTO tt VALUES('a x x x x');
+  INSERT INTO tt VALUES('x x a a a');
+  INSERT INTO tt VALUES('x a a x x');
+}
+
+proc firstinst {cmd} { 
+  foreach {p c o} [$cmd xInst 0] {}
+  return $o
+}
+sqlite3_fts5_create_function db firstinst firstinst
+
+do_execsql_test 2.1 {
+  SELECT rowid FROM tt('a') ORDER BY rank;
+} {2 3 1}
+
+do_execsql_test 2.2 {
+  SELECT rowid FROM tt('a', 'firstinst()') ORDER BY rank;
+} {1 3 2}
+
+do_execsql_test 2.3 {
+  INSERT INTO tt(tt, rank) VALUES('rank', 'firstinst()');
+  SELECT rowid FROM tt('a') ORDER BY rank;
+} {1 3 2}
+
+do_test 2.4 {
+  sqlite3 db2 test.db
+  catchsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
+} {1 {no such function: firstinst}}
+
+do_test 2.5 {
+  db2 close
+  sqlite3 db2 test.db
+  sqlite3_fts5_create_function db2 firstinst firstinst
+  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
+} {1 3 2}
+
+do_test 2.6 {
+  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
+} {1 3 2}
+
+do_test 2.7 {
+  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db
+} {1 3 2}
+
+
+
+
+
+
+
 finish_test

@@ -63,6 +63,7 @@ do_execsql_test 2.2 {
    SELECT rnddoc(6), rnddoc(6) FROM r
  )
  INSERT INTO x1 SELECT * FROM r LIMIT 10000;
+  DELETE FROM x1 WHERE (rowid%2);
 }

 set res [db one {SELECT count(*) FROM x1_data}]
@@ -71,8 +72,7 @@ do_execsql_test 2.3 {
 } $res
 do_execsql_test 2.4 {
  UPDATE x1_data SET block = X'';
-  -- SELECT count(fts5_decode(rowid, block)) FROM x1_data;
-  SELECT count(*) FROM x1_data;
+  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
 } $res

 do_execsql_test 2.5 {
@@ -184,5 +184,36 @@ do_execsql_test 5.2 {
  SELECT count(fts5_decode(rowid, block)) FROM x4_data;
 } $res

+#-------------------------------------------------------------------------
+#
+
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE x5 USING fts5(x, detail=none);
+  INSERT INTO x5(x5, rank) VALUES('pgsz', 32);
+  INSERT INTO x5 VALUES('a b c d e f');
+  INSERT INTO x5 VALUES('a b c d e f');
+  INSERT INTO x5 VALUES('a b c d e f');
+  BEGIN;
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+    ) INSERT INTO x5 SELECT 'a b c d e f' FROM s;
+  COMMIT;
+  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
+} {32}
+
+do_execsql_test 6.1 {
+  DELETE FROM x5 WHERE rowid <= 2;
+  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
+} {34}
+
+do_execsql_test 6.2 {
+  UPDATE x5 SET x='a b c d e f' WHERE rowid=3;
+  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
+} {36}
+
+#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM x5_data} {puts $r}
+
+
+
 finish_test

@@ -19,6 +19,8 @@ ifcapable !fts5 {
  return
 }

+if 1 {
+
 #-------------------------------------------------------------------------
 #
 set doc "x x [string repeat {y } 50]z z"
@@ -300,5 +302,102 @@ do_execsql_test 12.2 {
  COMMIT;
 } {}

+#-------------------------------------------------------------------------
+# Try an UPDATE OR REPLACE query.
+#
+do_execsql_test 13.1 {
+  CREATE VIRTUAL TABLE xy USING fts5(x);
+  INSERT INTO xy(rowid, x) VALUES(1, '1 2 3');
+  INSERT INTO xy(rowid, x) VALUES(2, '2 3 4');
+  INSERT INTO xy(rowid, x) VALUES(3, '3 4 5');
+}
+
+do_execsql_test 13.2 {
+  UPDATE OR REPLACE xy SET rowid=3 WHERE rowid = 2;
+  SELECT rowid, x FROM xy;
+} {
+  1 {1 2 3}
+  3 {2 3 4}
+}
+
+do_execsql_test 13.3 {
+  INSERT INTO xy(xy) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 14.1 {
+  CREATE VIRTUAL TABLE ttt USING fts5(x);
+  BEGIN;
+    INSERT INTO ttt(rowid, x) VALUES(1, 'a b c');
+    INSERT INTO ttt(rowid, x) VALUES(2, 'a b c');
+    INSERT INTO ttt(rowid, x) VALUES(3, 'a b c');
+  COMMIT;
+}
+do_test 14.2 { 
+  fts5_level_segs ttt 
+} {1}
+
+#-------------------------------------------------------------------------
+db func rnddoc fts5_rnddoc
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x);
+  INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
+
+  WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 )
+  INSERT INTO x1 SELECT rnddoc(5) FROM ii;
+}
+
+do_execsql_test 4.1 {
+  SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'
+} {0 {} 4}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 15.0 {
+  CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1);
+  INSERT INTO x2 VALUES('ab');
+}
+
+do_execsql_test 15.1 {
+  INSERT INTO x2(x2) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+foreach_detail_mode $testprefix {
+  reset_db
+  fts5_aux_test_functions db
+  do_execsql_test 16.0 {
+    CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%);
+    INSERT INTO x3 VALUES('a b c d e f');
+  }
+  do_execsql_test 16.1 {
+    SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c');
+  } {2.0.2}
+
+  do_execsql_test 16.1 {
+    SELECT fts5_test_poslist(x3) FROM x3('a OR c');
+  } {{0.0.0 1.0.2}}
+}
+
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 17.0 {
+  CREATE VIRTUAL TABLE x3 USING fts5(x);
+  INSERT INTO x3 VALUES('a b c');
+}
+
+do_execsql_test 17.1 {
+  SELECT rowid FROM x3('b AND d');
+}
+
+#-------------------------------------------------------------------------
+do_execsql_test 18.1 {
+  CREATE VIRTUAL TABLE x4 USING fts5(x);
+  SELECT rowid FROM x4('""');
+}
+
 finish_test

@@ -0,0 +1,338 @@
+# 2015 September 05
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5simple2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  INSERT INTO t1 VALUES('a b c');
+}
+do_execsql_test 1.1 {
+  SELECT rowid FROM t1('c a b')
+} {1}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  BEGIN;
+    INSERT INTO t1 VALUES('b c d');
+    INSERT INTO t1 VALUES('b c d');
+  COMMIT;
+}
+do_execsql_test 2.1 {
+  SELECT rowid FROM t1('b c d')
+} {1 2}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  BEGIN;
+    INSERT INTO t1 VALUES('b c d');
+    INSERT INTO t1 VALUES('b c d');
+}
+do_execsql_test 3.1 {
+  SELECT rowid FROM t1('b c d'); COMMIT;
+} {1 2}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  BEGIN;
+    INSERT INTO t1 VALUES('a1 b1 c1');
+    INSERT INTO t1 VALUES('a2 b2 c2');
+    INSERT INTO t1 VALUES('a3 b3 c3');
+  COMMIT;
+}
+do_execsql_test 4.1 {
+  SELECT rowid FROM t1('b*');
+} {1 2 3}
+
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  BEGIN;
+  INSERT INTO t1 VALUES('a1 b1 c1');
+  INSERT INTO t1 VALUES('a2 b2 c2');
+  INSERT INTO t1 VALUES('a1 b1 c1');
+  COMMIT;
+}
+do_execsql_test 5.1 { SELECT rowid FROM t1('b*') } {1 2 3}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=full);
+  BEGIN;
+  INSERT INTO t1 VALUES('a1 b1 c1');
+  INSERT INTO t1 VALUES('a1 b1 c1');
+  INSERT INTO t1 VALUES('a1 b1 c1');
+  COMMIT;
+}
+
+do_execsql_test 6.1 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 2 1}
+do_execsql_test 6.2 { SELECT rowid FROM t1('b1') ORDER BY rowid DESC } {3 2 1}
+do_execsql_test 6.3 { SELECT rowid FROM t1('c1') ORDER BY rowid DESC } {3 2 1}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  BEGIN;
+  INSERT INTO t1 VALUES('a1 b1');
+  INSERT INTO t1 VALUES('a1 b2');
+  COMMIT;
+}
+do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1}
+do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  INSERT INTO t1 VALUES('a1 b1 c1');
+  INSERT INTO t1 VALUES('a2 b2 c2');
+  INSERT INTO t1 VALUES('a1 b1 c1');
+}
+do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3}
+do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3}
+do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2}
+
+do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1}
+do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1}
+do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2}
+
+do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') }
+
+do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3}
+do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3}
+do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2}
+
+do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1}
+do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1}
+do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2}
+
+#--------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 9.0.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  INSERT INTO t1 VALUES('a1 b1 c1');
+  INSERT INTO t1 VALUES('a2 b2 c2');
+  INSERT INTO t1 VALUES('a1 b1 c1');
+}
+do_execsql_test 9.0.1 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {}
+
+reset_db
+do_execsql_test 9.1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none);
+  INSERT INTO t1 VALUES('a1 b1 c1', 'x y z');
+  INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3');
+  INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z');
+}
+do_execsql_test 9.2.1 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {}
+
+#--------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 10.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  INSERT INTO t1 VALUES('b1');
+  INSERT INTO t1 VALUES('b1');
+  DELETE FROM t1 WHERE rowid=1;
+}
+
+do_execsql_test 10.1 {
+  SELECT rowid FROM t1('b1');
+} {2}
+
+do_execsql_test 10.2 {
+  SELECT rowid FROM t1('b1') ORDER BY rowid DESC;
+} {2}
+
+do_execsql_test 10.3 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {}
+
+#--------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 11.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=none);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  WITH d(x,y) AS (
+    SELECT NULL, 'xyz' UNION ALL SELECT NULL, 'xyz' FROM d
+  )
+  INSERT INTO t1 SELECT * FROM d LIMIT 23;
+}
+
+#db eval { SELECT rowid AS r, quote(block) AS b FROM t1_data } { puts "$r: $b" }
+do_execsql_test 11.2 {
+  SELECT rowid FROM t1;
+} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23}
+
+do_execsql_test 11.3 {
+  SELECT rowid FROM t1('xyz');
+} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23}
+
+do_execsql_test 11.4 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 12.0 {
+  CREATE VIRTUAL TABLE yy USING fts5(x, detail=none);
+  INSERT INTO yy VALUES('in if');
+  INSERT INTO yy VALUES('if');
+} {}
+
+do_execsql_test 12.1 {
+  SELECT rowid FROM yy('i*');
+} {1 2}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 13.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, prefix=1, detail=none);
+} {}
+foreach {rowid a} {
+  0   {f}
+  1   {u}
+  2   {k}
+  3   {a}
+  4   {a}
+  5   {u}
+  6   {u}
+  7   {u}
+  8   {f}
+  9   {f}
+  10  {a}
+  11  {p}
+  12  {f}
+  13  {u}
+  14  {a}
+  15  {a}
+} {
+  do_execsql_test 13.1.$rowid {
+    INSERT INTO t1(rowid, a) VALUES($rowid, $a);
+  }
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+fts5_aux_test_functions db
+do_execsql_test 14.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  INSERT INTO t1 VALUES('a b c d');
+} {}
+
+do_execsql_test 14.1 {
+  SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
+} {0.0.1}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 15.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none);
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(1, 'sqlite');
+    INSERT INTO t1(rowid, x) VALUES(2, 'sqlite'); 
+  COMMIT;
+} {}
+
+do_test 15.1 {
+  execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+} {}
+
+do_test 15.2 {
+  execsql { DELETE FROM t1 }
+} {}
+
+do_execsql_test 15.3.1 {
+  SELECT rowid FROM t1('sqlite');
+} {}
+
+do_execsql_test 15.3.2 {
+  SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC;
+} {}
+
+do_test 15.4 {
+  execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+} {}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 16.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
+  BEGIN;
+    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
+    INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
+    INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
+  COMMIT;
+  UPDATE t2 SET x=x;
+}
+
+do_execsql_test 16.1 {
+  INSERT INTO t2(t2) VALUES('integrity-check');
+} {}
+
+do_execsql_test 16.2 {
+  SELECT rowid FROM t2('b') ORDER BY rowid DESC
+} {1000 456 1}
+
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 16.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
+  BEGIN;
+    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
+    INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
+    INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
+  COMMIT;
+  UPDATE t2 SET x=x;
+  DELETE FROM t2;
+}
+
+#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r}
+  
+finish_test
+
@@ -0,0 +1,85 @@
+# 2015 September 05
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5simple3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+fts5_aux_test_functions db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col);
+  INSERT INTO t1 VALUES('a', 'b', 'c');
+  INSERT INTO t1 VALUES('x', 'x', 'x');
+}
+
+do_execsql_test 1.1 {
+  SELECT rowid, fts5_test_collist(t1) FROM t1('a:a');
+} {1 0.0}
+
+do_execsql_test 1.2 {
+  SELECT rowid, fts5_test_collist(t1) FROM t1('b:x');
+} {2 0.1}
+
+do_execsql_test 1.3 {
+  SELECT rowid, fts5_test_collist(t1) FROM t1('b:a');
+} {}
+
+#-------------------------------------------------------------------------
+# Create detail=col and detail=full tables with 998 columns.
+#
+foreach_detail_mode $testprefix {
+  if {[detail_is_none]} continue
+
+  do_test 2.1 {
+    execsql { DROP TABLE IF EXISTS t2 }
+    set cols [list]
+    set vals [list]
+    for {set i 1} {$i <= 998} {incr i} {
+      lappend cols "c$i"
+      lappend vals "'val$i'"
+    }
+    execsql "CREATE VIRTUAL TABLE t2 USING fts5(detail=%DETAIL%,[join $cols ,])"
+  } {}
+  
+  do_test 2.2 {
+    execsql "INSERT INTO t2 VALUES([join $vals ,])"
+  } {}
+  
+  foreach {tn q res} {
+    1 { c1:val1 }     1
+    2 { c300:val300 } 1
+    3 { c300:val1 } {}
+    4 { c1:val300 } {}
+  } {
+    do_execsql_test 2.3.$tn {
+      SELECT rowid FROM t2($q)
+    } $res
+  }
+}
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x3 USING fts5(one);
+  INSERT INTO x3 VALUES('a b c');
+  INSERT INTO x3 VALUES('c b a');
+  INSERT INTO x3 VALUES('o t t');
+  SELECT * FROM x3('x OR y OR z');
+}
+
+
+finish_test
+
@@ -21,42 +21,16 @@ ifcapable !fts5 {
  return
 }

-foreach S {
-  {zero 0}
-  {one 1 i}
-  {two 2 ii}
-  {three 3 iii}
-  {four 4 iv}
-  {five 5 v}
-  {six 6 vi}
-  {seven 7 vii}
-  {eight 8 viii}
-  {nine 9 ix}
-} {
-  foreach s $S {
-    set o [list]
-    foreach x $S {if {$x!=$s} {lappend o $x}}
-    set ::syn($s) $o
-  }
-}
+proc tcl_create {args} { return "tcl_tokenize" }

-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-  }
-}
-
-proc tcl_create {args} {
-  return "tcl_tokenize"
-}
-
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+foreach_detail_mode $testprefix {

 #-------------------------------------------------------------------------
 # Warm body test for the code in fts5_tcl.c.
 #
+fts5_tclnum_register db
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = "tclnum document", detail=%DETAIL%);
  INSERT INTO ft VALUES('abc def ghi');
  INSERT INTO ft VALUES('jkl mno pqr');
  SELECT rowid, x FROM ft WHERE ft MATCH 'def';
@@ -67,22 +41,13 @@ do_execsql_test 1.0 {
 # Test a tokenizer that supports synonyms by adding extra entries to the
 # FTS index.
 #
-
-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-    if {$tflags=="document" && [info exists ::syn($w)]} {
-      foreach s $::syn($w) {
-        sqlite3_fts5_token -colo $s $iStart $iEnd
-      }
-    }
-  }
-}
 reset_db
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+fts5_tclnum_register db

 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize = "tclnum document", detail=%DETAIL%
+  );
  INSERT INTO ft VALUES('one two three');
  INSERT INTO ft VALUES('four five six');
  INSERT INTO ft VALUES('eight nine ten');
@@ -95,6 +60,7 @@ foreach {tn expr res} {
  4 "1*" {1}
  5 "1 + 2" {1}
 } {
+  if {![fts5_expr_ok $expr ft]} continue
  do_execsql_test 2.1.$tn {
    SELECT rowid FROM ft WHERE ft MATCH $expr
  } $res
@@ -180,17 +146,7 @@ do_execsql_test 3.2.5 {
 # Check that expressions with synonyms can be parsed and executed.
 #
 reset_db
-sqlite3_fts5_create_tokenizer db tcl tcl_create
-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-    if {$tflags=="query" && [info exists ::syn($w)]} {
-      foreach s $::syn($w) {
-        sqlite3_fts5_token -colo $s $iStart $iEnd
-      }
-    }
-  }
-}
+fts5_tclnum_register db

 foreach {tn expr res} {
  1  {abc}                           {"abc"}
@@ -198,11 +154,13 @@ foreach {tn expr res} {
  3  {3}                             {"3"|"iii"|"three"}
  4  {3*}                            {"3"|"iii"|"three" *}
 } {
-  do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res]
+  do_execsql_test 4.1.$tn {
+    SELECT fts5_expr($expr, 'tokenize=tclnum')
+  } [list $res]
 }

 do_execsql_test 4.2.1 {
-  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl);
+  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tclnum, detail=%DETAIL%);
  INSERT INTO xx VALUES('one two');
  INSERT INTO xx VALUES('three four');
 }
@@ -217,7 +175,7 @@ do_execsql_test 4.2.3 {

 do_test 5.0 {
  execsql { 
-    CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl)
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tclnum, detail=%DETAIL%)
  }
  foreach {rowid a b} {
    1 {four v 4 i three} {1 3 five five 4 one}
@@ -285,6 +243,7 @@ foreach {tn q res} {
    5 {three i v i four 4 1} {ii [five five five] iii}
  }
 } {
+  if {![fts5_expr_ok $q t1]} continue
  do_execsql_test 5.1.$tn {
    SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']')
    FROM t1 WHERE t1 MATCH $q
@@ -316,7 +275,6 @@ foreach {tn q res} {
  } $res
 }

-
 #-------------------------------------------------------------------------
 # Test terms with more than 4 synonyms.
 #
@@ -334,17 +292,19 @@ proc tcl_tokenize {tflags text} {
 }

 do_execsql_test 6.0.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl, detail=%DETAIL%);
  INSERT INTO t1 VALUES('yy xx qq');
  INSERT INTO t1 VALUES('yy xx xx');
 }
-do_execsql_test 6.0.2 {
-  SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
-} {{yy xx qq}}
+if {[fts5_expr_ok "NEAR(y q)" t1]} {
+  do_execsql_test 6.0.2 {
+    SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
+  } {{yy xx qq}}
+}

 do_test 6.0.3 {
  execsql { 
-    CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl)
+    CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl, detail=%DETAIL%)
  }
  foreach {rowid a b} {
    1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa}
@@ -387,6 +347,8 @@ foreach {tn q res} {
    2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]}
  }
 } {
+  if {![fts5_expr_ok $q t2]} continue
+
  do_execsql_test 6.1.$tn.asc {
    SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']')
    FROM t2 WHERE t2 MATCH $q
@@ -435,7 +397,7 @@ proc tcl_tokenize {tflags text} {
 }

 do_execsql_test 7.0.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl, detail=%DETAIL%);
  INSERT INTO t1 VALUES('0 2 3', '4 5 6 7');
  INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
  SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0';
@@ -446,7 +408,7 @@ do_execsql_test 7.0.2 {
 }

 do_execsql_test 7.1.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl, detail=%DETAIL%);
  INSERT INTO t2 VALUES('0 2 3', '4 5 6 7');
  INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
  SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0';
@@ -456,5 +418,7 @@ do_execsql_test 7.1.2 {
  INSERT INTO t2(t2) VALUES('integrity-check');
 }

+} ;# foreach_detail_mode
+
 finish_test

@@ -0,0 +1,164 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on custom tokenizers that support synonyms.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5synonym2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach tok {query document} {
+foreach_detail_mode $testprefix {
+
+fts5_tclnum_register db
+fts5_aux_test_functions db
+
+proc fts5_test_bothlist {cmd} {
+
+  for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
+    set bFirst 1
+    $cmd xPhraseColumnForeach $i c { 
+      lappend CL $i.$c 
+      if {$bFirst} { $cmd xPhraseForeach $i c o { lappend PL $i.$c.$o } }
+      set bFirst 0
+    }
+  }
+
+  list [sort_poslist $PL] $CL
+}
+sqlite3_fts5_create_function db fts5_test_bothlist fts5_test_bothlist
+
+proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] }
+sqlite3_fts5_create_function db fts5_rowid fts5_rowid
+
+do_execsql_test 1.$tok.0.1 "
+  CREATE VIRTUAL TABLE ss USING fts5(a, b, 
+       tokenize='tclnum $tok', detail=%DETAIL%);
+  INSERT INTO ss(ss, rank) VALUES('rank', 'fts5_rowid()');
+"
+
+do_execsql_test 1.$tok.0.2 {
+  INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i');
+  INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi');
+  INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3');
+  INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7');
+  INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii');
+  INSERT INTO ss VALUES('one 7 seven six 2 two', '1 2 four 7 4 3 4');
+  INSERT INTO ss VALUES('eight iv 4 nine vii six 1', '5 6 v one zero 4');
+  INSERT INTO ss VALUES('v 9 8 iii 4', '9 4 seven two vi vii');
+  INSERT INTO ss VALUES('3 ix two 9 0 nine i', 'five ii nine two viii i five');
+  INSERT INTO ss VALUES('six iii 9 two eight 2', 'nine i nine vii nine');
+  INSERT INTO ss VALUES('6 three zero seven vii five', '8 vii ix 0 7 seven');
+  INSERT INTO ss VALUES('8 vii 8 7 3 4', 'eight iii four viii nine iv three');
+  INSERT INTO ss VALUES('4 v 7 two 0 one 8', 'vii 1 two five i zero 9');
+  INSERT INTO ss VALUES('3 ii vii vi eight', '8 4 ix one three eight');
+  INSERT INTO ss VALUES('iv eight seven 6 9 seven', 'one vi two five seven');
+  INSERT INTO ss VALUES('i i 5 i v vii eight', '2 seven i 2 2 four');
+  INSERT INTO ss VALUES('0 i iii nine 3 ix five', '0 eight iv 0 six 2');
+  INSERT INTO ss VALUES('iv vii three 3 9 one 8', '2 ii 6 eight ii six six');
+  INSERT INTO ss VALUES('eight one two nine six', '8 9 3 viii vi');
+  INSERT INTO ss VALUES('one 0 four ii eight one 3', 'iii eight vi vi vi');
+  INSERT INTO ss VALUES('4 0 eight 0 0', '1 four one vii seven ii');
+  INSERT INTO ss VALUES('1 zero nine 2 2', 'viii iv two vi nine v iii');
+  INSERT INTO ss VALUES('5 five viii four four vi', '8 five 7 vii 6 4');
+  INSERT INTO ss VALUES('7 ix four 8 vii', 'nine three nine ii ix vii');
+  INSERT INTO ss VALUES('nine iv v i 0 v', 'two iv vii six i ix 4');
+  INSERT INTO ss VALUES('one v v one viii 3 8', '2 1 3 five iii');
+  INSERT INTO ss VALUES('six ii 5 nine 4 viii seven', 'eight i ix ix 7 four');
+  INSERT INTO ss VALUES('9 ii two seven three 7 0', 'six viii seven 7 five');
+  INSERT INTO ss VALUES('five two 4 viii nine', '9 7 nine zero 1 two one');
+  INSERT INTO ss VALUES('viii 8 iii i ii 8 3', '4 2 7 v 8 8');
+  INSERT INTO ss VALUES('four vii 4 iii zero 0 vii', '3 viii iii zero 9 i');
+  INSERT INTO ss VALUES('0 seven v five i five v', 'one 4 2 ix 9');
+  INSERT INTO ss VALUES('two 5 two two ix 4 1', '3 nine ii v nine 3 five');
+  INSERT INTO ss VALUES('five 5 7 4 6 vii', 'three 2 ix 2 8 6');
+  INSERT INTO ss VALUES('six iii vi iv seven eight', '8 six 7 0 4');
+  INSERT INTO ss VALUES('vi vi iv 3 0 one one', '9 6 eight ix iv');
+  INSERT INTO ss VALUES('7 2 2 iii 0', '0 0 seven 1 nine');
+  INSERT INTO ss VALUES('8 6 iv six ii', 'iv 6 3 4 ii five');
+  INSERT INTO ss VALUES('0 two two seven ii', 'vii ix four 4 zero vi vi');
+  INSERT INTO ss VALUES('2 one eight 8 9 7', 'vi 3 0 3 vii');
+  INSERT INTO ss VALUES('iii ii ix iv three', 'vi i 6 1 two');
+  INSERT INTO ss VALUES('eight four nine 8 seven', 'one three i nine iii one');
+  INSERT INTO ss VALUES('iii seven five ix 8', 'ii 7 seven 0 four ii');
+  INSERT INTO ss VALUES('four 0 1 5 two', 'iii 9 5 ii ii 2 4');
+  INSERT INTO ss VALUES('iii nine four vi 8 five six', 'i i ii seven vi vii');
+  INSERT INTO ss VALUES('eight vii eight six 3', 'i vii 1 six 9 vii');
+  INSERT INTO ss VALUES('9 0 viii viii five', 'i 1 viii ix 3 4');
+  INSERT INTO ss VALUES('three nine 5 nine viii four zero', 'ii i 1 5 2 viii');
+  INSERT INTO ss VALUES('5 vii three 9 four', 'three five one 7 2 eight one');
+}
+
+foreach {tn expr} {
+  2.1 "one OR two OR three OR four"
+
+  1.1 "one"   1.2 "two"   1.3 "three"   1.4 "four"
+  1.5 "v"     1.6 "vi"    1.7 "vii"     1.8 "viii"
+  1.9 "9"    1.10 "0"    1.11 "1"      1.12 "2"
+
+  2.1 "one OR two OR three OR four"
+  2.2 "(one AND two) OR (three AND four)"
+  2.3 "(one AND two) OR (three AND four) NOT five"
+  2.4 "(one AND two) NOT 6"
+
+  3.1 "b:one AND a:two"
+  3.2 "b:one OR a:two"
+  3.3 "a:one OR b:1 OR {a b} : i"
+
+  4.1 "NEAR(one two, 2)"
+  4.2 "NEAR(one two three, 2)"
+  4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)"
+} {
+  if {[fts5_expr_ok $expr ss]==0} {
+    do_test 1.$tok.$tn.OMITTED { list } [list]
+    continue
+  }
+
+  set res [fts5_query_data $expr ss ASC ::tclnum_syn]
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.1 {
+    SELECT rowid, fts5_test_poslist2(ss), fts5_test_collist(ss) FROM ss($expr)
+  } $res
+
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.2 {
+    SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
+  } $res
+
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.2 {
+    SELECT rowid, fts5_test_poslist2(ss), fts5_test_collist(ss) FROM ss($expr)
+    ORDER BY rank ASC
+  } $res
+
+  set res2 [list]
+  foreach {a b c} $res { lappend res2 $a $c $b }
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.3 {
+    SELECT rowid, fts5_test_collist(ss), fts5_test_poslist2(ss) FROM ss($expr)
+  } $res2
+
+  set res3 [list]
+  foreach {a b c} $res { lappend res3 $a [list $b $c] }
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.3 {
+    SELECT rowid, fts5_test_bothlist(ss) FROM ss($expr)
+  } $res3
+
+
+}
+
+}
+}
+
+finish_test
+
@@ -0,0 +1,115 @@
+# 2016 Jan 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5tok1
+
+
+sqlite3_fts5_register_fts5tokenize db
+
+#-------------------------------------------------------------------------
+# Simple test cases. Using the default (ascii) tokenizer.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5tokenize(ascii);
+  CREATE VIRTUAL TABLE t2 USING fts5tokenize();
+  CREATE VIRTUAL TABLE t3 USING fts5tokenize(
+      ascii, 'separators', 'xyz', tokenchars, ''''
+  );
+}
+
+foreach {tn tbl} {1 t1 2 t2 3 t3} {
+  do_execsql_test 1.$tn.1 "SELECT input, * FROM $tbl ('one two three')" {
+    {one two three} one   0  3 0 
+    {one two three} two   4  7 1 
+    {one two three} three 8 13 2
+  }
+
+  do_execsql_test 1.$tn.2 "
+    SELECT token FROM $tbl WHERE input = 'OnE tWo tHrEe'
+  " {
+    one two three
+  }
+}
+
+do_execsql_test 1.4 {
+  SELECT token FROM t3 WHERE input = '1x2x3x'
+} {1 2 3}
+
+do_execsql_test 1.5 {
+  SELECT token FROM t1 WHERE input = '1x2x3x'
+} {1x2x3x}
+
+do_execsql_test 1.6 {
+  SELECT token FROM t3 WHERE input = '1''2x3x'
+} {1'2 3}
+
+do_execsql_test 1.7 {
+  SELECT token FROM t3 WHERE input = ''
+} {}
+
+do_execsql_test 1.8 {
+  SELECT token FROM t3 WHERE input = NULL
+} {}
+
+do_execsql_test 1.9 {
+  SELECT input, * FROM t3 WHERE input = 123
+} {123 123 0 3 0}
+
+do_execsql_test 1.10 {
+  SELECT input, * FROM t1 WHERE input = 'a b c' AND token = 'b';
+} {
+  {a b c} b 2 3 1
+}
+
+do_execsql_test 1.11 {
+  SELECT input, * FROM t1 WHERE token = 'b' AND input = 'a b c';
+} {
+  {a b c} b 2 3 1
+}
+
+do_execsql_test 1.12 {
+  SELECT input, * FROM t1 WHERE input < 'b' AND input = 'a b c';
+} {
+  {a b c} a 0 1 0 
+  {a b c} b 2 3 1 
+  {a b c} c 4 5 2
+}
+
+do_execsql_test 1.13.1 {
+  CREATE TABLE c1(x);
+  INSERT INTO c1(x) VALUES('a b c');
+  INSERT INTO c1(x) VALUES('d e f');
+}
+do_execsql_test 1.13.2 {
+  SELECT c1.*, input, t1.* FROM c1, t1 WHERE input = x AND c1.rowid=t1.rowid;
+} {
+  {a b c} {a b c} a 0 1 0 
+  {d e f} {d e f} e 2 3 1 
+}
+
+
+#-------------------------------------------------------------------------
+# Error cases.
+#
+do_catchsql_test 2.0 {
+  CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer);
+} {1 {vtable constructor failed: tX}}
+
+do_catchsql_test 2.1 {
+  CREATE VIRTUAL TABLE t4 USING fts5tokenize;
+  SELECT * FROM t4;
+} {1 {SQL logic error or missing database}}
+
+
+finish_test
@@ -0,0 +1,47 @@
+# 2016 Jan 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5||!fts3 { finish_test ; return }
+set ::testprefix fts5tok2
+
+sqlite3_fts5_register_fts5tokenize db
+
+#-------------------------------------------------------------------------
+# Simple test cases. Using the default (ascii) tokenizer.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t5 USING fts5tokenize(unicode61);
+  CREATE VIRTUAL TABLE t3 USING fts3tokenize(unicode61);
+}
+
+do_test 1.1 {
+  array unset -nocomplain A
+
+  for {set i 1} {$i < 65536} {incr i} {
+    set input [format "abc%cxyz" $i]
+      set expect [execsql {
+        SELECT input, token, start, end FROM t3 WHERE input=$input
+    }]
+
+    incr A([llength $expect])
+
+    set res [execsql {
+      SELECT input, token, start, end FROM t5($input)
+    }]
+    if {$res != $expect} {error "failed at i=$i"}
+  }
+} {}
+
+do_test 1.1.nTokenChars=$A(4).nSeparators=$A(8) {} {}
+
+finish_test
@@ -0,0 +1,121 @@
+# 2016 Jan 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5update
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set docs {
+  "eight zero iv eight 7"            "ix one 8 one three ii one"        
+  "1 9 9 three viii"                 "5 zero ii 6 nine ix 3"            
+  "3 zero 5 2 seven nine"            "two eight viii eight 1"           
+  "4 six two 5 9 vii"                "viii ii four 8 i i iv"            
+  "vii 0 iv seven 7 viii"            "five 1 nine vi seven"             
+  "1 zero zero iii 1"                "one one six 6 nine seven"         
+  "one v 4 zero 4 iii ii"            "2 3 eight six ix"                 
+  "six iv 7 three 5"                 "ix zero 0 8 ii 7 3"               
+  "four six nine 2 vii 3"            "five viii 5 8 0 7"                
+}
+
+foreach_detail_mode $::testprefix {
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+} {}
+
+do_test 1.1 {
+  foreach {a b} $docs {
+    execsql {INSERT INTO t1 VALUES($a, $b)}
+  }
+} {}
+
+proc update {iRowid iA iB} {
+  set a [lindex $::docs $iA]
+  set b [lindex $::docs $iB]
+  execsql { UPDATE t1 SET a=$a, b=$b WHERE rowid=$iRowid }
+}
+
+set nDoc [llength $::docs]
+foreach n {1 5 10 50 100} {
+  do_test 1.2.$n {
+    execsql BEGIN
+    for {set i 1} {$i <= 1000} {incr i} {
+      set iRowid [expr {int(rand() * ($nDoc/2)) + 1}]
+      set iA [expr {int(rand() * $nDoc)}]
+      set iB [expr {int(rand() * $nDoc)}]
+      update $iRowid $iA $iB
+
+      if {($i % $n)==0} {
+        execsql { COMMIT; BEGIN }
+      }
+
+      if {($i % $n)==100} {
+        execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+      }
+    }
+    execsql COMMIT
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+  } {}
+}
+
+do_execsql_test 1.3 {
+  UPDATE t1 SET a=a AND b=b;
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_test 1.4 {
+  execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) }
+  for {set i 0} {$i < 50} {incr i} {
+    execsql { UPDATE t1 SET a=a AND b=b }
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+  }
+} {}
+
+#-------------------------------------------------------------------------
+# Lots of deletes/inserts of the same document with the same rowid.
+#
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE x2 USING fts5(x, detail=%DETAIL%);
+  INSERT INTO x2(x2, rank) VALUES('crisismerge', 2);
+  INSERT INTO x2 VALUES('a b c');
+  INSERT INTO x2 VALUES('a b c');
+}
+do_test 2.1 {
+  for {set i 0} {$i < 1000} {incr i} {
+    execsql { DELETE FROM x2 WHERE rowid = 2 }
+    execsql { INSERT INTO x2(rowid, x) VALUES(2, 'a b c') }
+  }
+} {}
+do_execsql_test 2.1.integrity {
+  INSERT INTO x2(x2) VALUES('integrity-check');
+}
+
+do_test 2.2 {
+  for {set i 0} {$i < 1000} {incr i} {
+    execsql { UPDATE x2 SET x=x WHERE rowid=2 }
+  }
+} {}
+do_execsql_test 2.2.integrity {
+  INSERT INTO x2(x2) VALUES('integrity-check');
+}
+
+}
+finish_test
+
+
@@ -21,9 +21,46 @@ ifcapable !fts5 {
  return
 }

+foreach_detail_mode $testprefix {
+
+proc null_list_entries {iFirst nInterval L} {
+  for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} {
+    lset L $i {}
+  }
+  return $L
+}
+
+proc star_from_row {L} {
+  if {[detail_is_full]==0} {
+    set L [null_list_entries 2 3 $L]
+  }
+  return $L
+}
+
+proc star_from_col {L} {
+  if {[detail_is_col]} {
+    set L [null_list_entries 3 4 $L]
+  }
+  if {[detail_is_none]} {
+    set L [null_list_entries 1 4 $L]
+    set L [null_list_entries 3 4 $L]
+  }
+  return $L
+}
+
+proc row_to_col {L} {
+  if {[detail_is_none]==0} { error "this is for detail=none mode" }
+  set ret [list]
+  foreach {a b c} $L {
+    lappend ret $a {} $b {}
+  }
+  set ret
+}
+
+if 1 {

 do_execsql_test 1.1.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1);
+  CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%);
  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row');
  PRAGMA table_info = v1;
 } {
@@ -52,32 +89,32 @@ do_execsql_test 1.3 {

 do_execsql_test 1.4.1 {
  SELECT * FROM v1;
-} {x 2 4  y 1 1  z 1 1}
+} [star_from_row {x 2 4  y 1 1  z 1 1}]

 do_execsql_test 1.4.2 {
  SELECT * FROM v2;
-} {x 0 2 4  y 0 1 1  z 0 1 1}
+} [star_from_col {x one 2 4  y one 1 1  z one 1 1}]

 do_execsql_test 1.5.1 {
  BEGIN;
    INSERT INTO t1 VALUES('a b c');
    SELECT * FROM v1 WHERE term<'d';
-} {a 1 1   b 1 1   c 1 1}
+} [star_from_row {a 1 1   b 1 1   c 1 1}]

 do_execsql_test 1.5.2 {
    SELECT * FROM v2 WHERE term<'d';
  COMMIT;
-} {a 0 1 1  b 0 1 1  c 0 1 1}
+} [star_from_col {a one 1 1  b one 1 1  c one 1 1}]

 do_execsql_test 1.6 {
  DELETE FROM t1 WHERE one = 'a b c';
  SELECT * FROM v1;
-} {x 2 4  y 1 1  z 1 1}
+} [star_from_row {x 2 4  y 1 1  z 1 1}]

 #-------------------------------------------------------------------------
 #
 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE tt USING fts5(a, b);
+  CREATE VIRTUAL TABLE tt USING fts5(a, b, detail=%DETAIL%);
  INSERT INTO tt VALUES('d g b f d f', 'f c e c d a');
  INSERT INTO tt VALUES('f a e a a b', 'e d c f d d');
  INSERT INTO tt VALUES('b c a a a b', 'f f c c b c');
@@ -90,20 +127,23 @@ do_execsql_test 2.0 {
  INSERT INTO tt VALUES('c c a a c f', 'd g a e b g');
 }

-set res_col {
-  a 0 6 11    a 1 7 9
-  b 0 6 7     b 1 7 7 
-  c 0 6 12    c 1 5 8 
-  d 0 4 6     d 1 9 13 
-  e 0 6 7     e 1 6 6 
-  f 0 9 10    f 1 7 10 
-  g 0 5 7     g 1 5 7
-  x 0 1 1     y 1 1 1
-}
-set res_row {
+set res_row [star_from_row {
  a 10 20   b 9 14   c 9 20   d 9 19   
  e 8 13   f 10 20   g 7 14   x 1 1   
  y 1 1
+}]
+set res_col [star_from_col {
+  a a 6 11    a b 7 9
+  b a 6 7     b b 7 7 
+  c a 6 12    c b 5 8 
+  d a 4 6     d b 9 13 
+  e a 6 7     e b 6 6 
+  f a 9 10    f b 7 10 
+  g a 5 7     g b 5 7
+  x a 1 1     y b 1 1
+}]
+if {[detail_is_none]} {
+  set res_col [row_to_col $res_row]
 }

 foreach {tn tbl resname} {
@@ -153,9 +193,9 @@ reset_db
 forcedelete test.db2
 do_execsql_test 5.0 {
  ATTACH 'test.db2' AS aux;
-  CREATE VIRTUAL TABLE t1 USING fts5(x);
-  CREATE VIRTUAL TABLE temp.t1 USING fts5(x);
-  CREATE VIRTUAL TABLE aux.t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE aux.t1 USING fts5(x, detail=%DETAIL%);

  INSERT INTO main.t1 VALUES('a b c');
  INSERT INTO main.t1 VALUES('d e f');
@@ -178,18 +218,18 @@ do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE temp.va  USING fts5vocab(aux, t1, row);
 }

-do_execsql_test 5.2 { SELECT * FROM vm } {
+do_execsql_test 5.2 { SELECT * FROM vm } [star_from_row {
  a 2 2 b 1 1 c 2 2 d 1 1 e 2 2 f 1 1
-}
-do_execsql_test 5.3 { SELECT * FROM vt1 } {
+}]
+do_execsql_test 5.3 { SELECT * FROM vt1 } [star_from_row {
  1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1
-}
-do_execsql_test 5.4 { SELECT * FROM vt2 } {
+}]
+do_execsql_test 5.4 { SELECT * FROM vt2 } [star_from_row {
  1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1
-}
-do_execsql_test 5.5 { SELECT * FROM va } {
+}]
+do_execsql_test 5.5 { SELECT * FROM va } [star_from_row {
  m 1 1 n 2 2 o 1 1 x 2 2 y 1 1 z 2 2
-}
+}]

 #-------------------------------------------------------------------------
 #
@@ -213,5 +253,197 @@ do_catchsql_test 6.2 {
  SELECT * FROM vocab3;
 } {1 {no such fts5 table: main.lll}}

+#-------------------------------------------------------------------------
+# Test single term queries on fts5vocab tables (i.e. those with term=?
+# constraints in the WHERE clause).
+#
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE tx USING fts5(one, two, detail=%DETAIL%);
+  INSERT INTO tx VALUES('g a ggg g a b eee',      'cc d aa ff g ee');
+  INSERT INTO tx VALUES('dd fff i a i jjj',       'f fff hh jj e f');
+  INSERT INTO tx VALUES('ggg a f f fff dd aa',    'd ggg f f j gg ddd');
+  INSERT INTO tx VALUES('e bb h jjj ii gg',       'e aa e f c fff');
+  INSERT INTO tx VALUES('j ff aa a h',            'h a j bbb bb');
+  INSERT INTO tx VALUES('cc i ff c d f',          'dd ii fff f c cc d');
+  INSERT INTO tx VALUES('jjj g i bb cc eee',      'hhh iii aaa b bbb aaa');
+  INSERT INTO tx VALUES('hhh hhh hhh bb fff f',   'fff gg aa ii h a');
+  INSERT INTO tx VALUES('b c cc aaa iii ggg f',   'iii ff ee a ff c cc');
+  INSERT INTO tx VALUES('hhh b hhh aaa j i i',    'dd ee ee aa bbb iii');
+  INSERT INTO tx VALUES('hh dd h b g ff i',       'ccc bb cc ccc f a d');
+  INSERT INTO tx VALUES('g d b ggg jj',           'fff jj ff jj g gg ee');
+  INSERT INTO tx VALUES('g ee ggg ggg cc bb eee', 'aa j jjj bbb dd eee ff');
+  INSERT INTO tx VALUES('c jjj hh ddd dd h',      'e aaa h jjj gg');
+
+  CREATE VIRTUAL TABLE txr USING fts5vocab(tx, row);
+  CREATE VIRTUAL TABLE txc USING fts5vocab(tx, col);
+}
+
+proc cont {L elem} {
+  set n 0
+  foreach e $L { if {$elem==$e} {incr n} }
+  set n
+}
+db func cont cont
+
+foreach {term} {
+  a aa aaa
+  b bb bbb
+  c cc ccc
+  d dd ddd
+  e ee eee
+  f ff fff
+  g gg ggg
+  h hh hhh
+  i ii iii
+  j jj jjj
+} {
+  set resr [db eval {
+    SELECT $term, 
+      sum(cont(one || ' ' || two, $term) > 0),
+      sum(cont(one || ' ' || two, $term))
+    FROM tx
+  }]
+  if {[lindex $resr 1]==0} {set resr [list]}
+
+  set r1 [db eval {
+    SELECT $term, 'one', sum(cont(one, $term)>0), sum(cont(one, $term)) FROM tx
+  }]
+  if {[lindex $r1 2]==0} {set r1 [list]}
+
+  set r2 [db eval {
+    SELECT $term, 'two', sum(cont(two, $term)>0), sum(cont(two, $term)) FROM tx
+  }]
+  if {[lindex $r2 2]==0} {set r2 [list]}
+
+  set resc [concat $r1 $r2]
+
+  set resc [star_from_col $resc]
+  set resr [star_from_row $resr]
+  if {[detail_is_none]} { set resc [row_to_col $resr] }
+  do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc
+  do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr
+}
+
+do_execsql_test 7.1 {
+  CREATE TABLE txr_c AS SELECT * FROM txr;
+  CREATE TABLE txc_c AS SELECT * FROM txc;
+}
+
+# Test range queries on the fts5vocab tables created above.
+#
+foreach {tn a b} {
+  1   a   jjj
+  2   bb  j
+  3   ccc ddd
+  4   dd  xyz
+  5   xzy dd
+  6   h   hh
+} {
+  do_execsql_test 7.2.$tn.1 {
+    SELECT * FROM txr WHERE term>=$a
+  } [db eval {SELECT * FROM txr_c WHERE term>=$a}]
+  do_execsql_test 7.2.$tn.2 {
+    SELECT * FROM txr WHERE term<=$b
+  } [db eval {SELECT * FROM txr_c WHERE term <=$b}]
+  do_execsql_test 7.2.$tn.3 {
+    SELECT * FROM txr WHERE term>=$a AND term<=$b
+  } [db eval {SELECT * FROM txr_c WHERE term>=$a AND term <=$b}]
+
+  do_execsql_test 7.2.$tn.4 {
+    SELECT * FROM txc WHERE term>=$a
+  } [db eval {SELECT * FROM txc_c WHERE term>=$a}]
+  do_execsql_test 7.2.$tn.5 {
+    SELECT * FROM txc WHERE term<=$b
+  } [db eval {SELECT * FROM txc_c WHERE term <=$b}]
+  do_execsql_test 7.2.$tn.6 {
+    SELECT * FROM txc WHERE term>=$a AND term<=$b
+  } [db eval {SELECT * FROM txc_c WHERE term>=$a AND term <=$b}]
+
+  do_execsql_test 7.2.$tn.7 {
+    SELECT * FROM txr WHERE term>$a
+  } [db eval {SELECT * FROM txr_c WHERE term>$a}]
+  do_execsql_test 7.2.$tn.8 {
+    SELECT * FROM txr WHERE term<$b
+  } [db eval {SELECT * FROM txr_c WHERE term<$b}]
+  do_execsql_test 7.2.$tn.9 {
+    SELECT * FROM txr WHERE term>$a AND term<$b
+  } [db eval {SELECT * FROM txr_c WHERE term>$a AND term <$b}]
+
+  do_execsql_test 7.2.$tn.10 {
+    SELECT * FROM txc WHERE term>$a
+  } [db eval {SELECT * FROM txc_c WHERE term>$a}]
+  do_execsql_test 7.2.$tn.11 {
+    SELECT * FROM txc WHERE term<$b
+  } [db eval {SELECT * FROM txc_c WHERE term<$b}]
+  do_execsql_test 7.2.$tn.12 {
+    SELECT * FROM txc WHERE term>$a AND term<$b
+  } [db eval {SELECT * FROM txc_c WHERE term>$a AND term <$b}]
+}
+
+do_execsql_test 7.3.1 {
+  SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term;
+} {30}
+
+if {![detail_is_none]} {
+  do_execsql_test 7.3.2 {
+    SELECT count(*) FROM txc, txc_c
+      WHERE txc.term = txc_c.term AND txc.col=txc_c.col;
+  } {57}
+}
+
+}
+
+#-------------------------------------------------------------------------
+# Test the fts5vocab tables response to a specific types of corruption:
+# where the fts5 index contains hits for columns that do not exist.
+#
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b, c, detail=%DETAIL%);
+  INSERT INTO x1 VALUES('a b c', 'd e f', 'g h i');
+  INSERT INTO x1 VALUES('g h i', 'a b c', 'd e f');
+  INSERT INTO x1 VALUES('d e f', 'g h i', 'a b c');
+  CREATE VIRTUAL TABLE x1_r USING fts5vocab(x1, row);
+  CREATE VIRTUAL TABLE x1_c USING fts5vocab(x1, col);
+}
+
+set resr [star_from_row {a 3 3 b 3 3 c 3 3 d 3 3 e 3 3 f 3 3 g 3 3 h 3 3 i 3 3}]
+set resc [star_from_col {
+  a a 1 1 a b 1 1 a c 1 1 b a 1 1 
+  b b 1 1 b c 1 1 c a 1 1 c b 1 1 
+  c c 1 1 d a 1 1 d b 1 1 d c 1 1
+  e a 1 1 e b 1 1 e c 1 1 f a 1 1 
+  f b 1 1 f c 1 1 g a 1 1 g b 1 1 
+  g c 1 1 h a 1 1 h b 1 1 h c 1 1 
+  i a 1 1 i b 1 1 i c 1 1
+}]
+if {[detail_is_none]} { set resc [row_to_col $resr] }
+
+do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
+do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc
+
+do_execsql_test 8.2 {
+  PRAGMA writable_schema = 1;
+  UPDATE sqlite_master 
+  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(a, detail=%DETAIL%)'
+  WHERE name = 'x1';
+}
+db close
+sqlite3 db test.db
+sqlite3_fts5_may_be_corrupt 1
+
+do_execsql_test 8.2.1 { SELECT * FROM x1_r } $resr
+
+if {[detail_is_none]} {
+  do_execsql_test 8.2.2 { SELECT * FROM x1_c } $resc
+} else {
+  do_catchsql_test 8.2.2 { 
+    SELECT * FROM x1_c 
+  } {1 {database disk image is malformed}}
+}
+
+sqlite3_fts5_may_be_corrupt 0
+
+}
+
 finish_test

@@ -0,0 +1,64 @@
+
+
+set Q {
+  {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron'"}
+  {25  "SELECT count(*) FROM t1 WHERE t1 MATCH 'hours'"}
+  {300 "SELECT count(*) FROM t1 WHERE t1 MATCH 'acid'"}
+  {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'loaned OR mobility OR popcore OR sunk'"}
+  {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron AND myapps'"}
+  {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'en* AND my*'"}
+
+  {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"}
+  {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"}
+  {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"}
+  {2   "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:the'"}
+
+  {2   "SELECT count(*) FROM t1 WHERE t1 MATCH 'd:holmes OR e:holmes OR f:holmes OR g:holmes'" }
+  {2   "SELECT count(*) FROM t1 WHERE t1 MATCH 'd:holmes AND e:holmes AND f:holmes AND g:holmes'" }
+  {4   "SELECT count(*) FROM t1 WHERE t1 MATCH 'd:holmes NOT e:holmes'" }
+}
+
+proc usage {} {
+  global Q
+  puts stderr "Usage: $::argv0 DATABASE QUERY"
+  puts stderr ""
+  for {set i 1} {$i <= [llength $Q]} {incr i} {
+    puts stderr "       $i. [lindex $Q [expr $i-1]]"
+  }
+  puts stderr ""
+  exit -1
+}
+
+
+set nArg [llength $argv]
+if {$nArg!=2 && $nArg!=3} usage
+set database [lindex $argv 0]
+set iquery [lindex $argv 1]
+if {$iquery<1 || $iquery>[llength $Q]} usage
+set nRepeat 0
+if {$nArg==3} { set nRepeat [lindex $argv 2] }
+
+
+sqlite3 db $database
+catch { load_static_extension db fts5 }
+
+incr iquery -1
+set sql [lindex $Q $iquery 1]
+if {$nRepeat==0} {
+  set nRepeat [lindex $Q $iquery 0]
+}
+
+puts "sql:     $sql"
+puts "nRepeat: $nRepeat"
+if {[regexp matchinfo $sql]} {
+  sqlite3_fts5_register_matchinfo db
+  db eval $sql 
+} else {
+  puts "result:  [db eval $sql]"
+}
+
+for {set i 1} {$i < $nRepeat} {incr i} {
+  db eval $sql
+}
+
+
@@ -1,95 +1,180 @@
-
-
-proc usage {} {
-  puts stderr "$::argv0 ?OPTIONS? DATABASE FILE1..."
-  puts stderr ""
-  puts stderr "Options are"
-  puts stderr "  -fts5"
-  puts stderr "  -fts4"
-  puts stderr "  -colsize <list of column sizes>"
-  puts stderr {
-This script is designed to create fts4/5 tables with more than one column.
-The -colsize option should be set to a Tcl list of integer values, one for
-each column in the table. Each value is the number of tokens that will be
-inserted into the column value for each row. For example, setting the -colsize
-option to "5 10" creates an FTS table with 2 columns, with roughly 5 and 10
-tokens per row in each, respectively.
-
-Each "FILE" argument should be a text file. The contents of these text files is
-split on whitespace characters to form a list of tokens. The first N1 tokens
-are used for the first column of the first row, where N1 is the first element
-of the -colsize list. The next N2 are used for the second column of the first
-row, and so on. Rows are added to the table until the entire list of tokens
-is exhausted.
-}
-  exit -1
-}
-
-set O(aColsize)       [list 10 10 10]
-set O(tblname)        t1
-set O(fts)            fts5
-
-
-set options_with_values {-colsize}
-
-for {set i 0} {$i < [llength $argv]} {incr i} {
-  set opt [lindex $argv $i]
-  if {[string range $opt 0 0]!="-"} break
-
-  if {[lsearch $options_with_values $opt]>=0} {
-    incr i
-    if {$i==[llength $argv]} usage
-    set val [lindex $argv $i]
-  }
-
-  switch -- $opt {
-    -colsize {
-      set O(aColSize) $val
-    }
-
-    -fts4 {
-      set O(fts) fts4
-    }
-
-    -fts5 {
-      set O(fts) fts5
-    }
+##########################################################################
+# 2016 Jan 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+proc process_cmdline {} { 
+  cmdline::process ::A $::argv {
+    {fts5                 "use fts5 (this is the default)"}
+    {fts4                 "use fts4"}
+    {colsize   "10 10 10" "list of column sizes"}
+    {tblname   "t1"       "table name to create"}
+    {detail    "full"     "Fts5 detail mode to use"}
+    {repeat    1          "Load each file this many times"}
+    {prefix    ""         "Fts prefix= option"}
+    database
+    file...
+  } {
+  This script is designed to create fts4/5 tables with more than one column.
+  The -colsize option should be set to a Tcl list of integer values, one for
+  each column in the table. Each value is the number of tokens that will be
+  inserted into the column value for each row. For example, setting the -colsize
+  option to "5 10" creates an FTS table with 2 columns, with roughly 5 and 10
+  tokens per row in each, respectively.
+  
+  Each "FILE" argument should be a text file. The contents of these text files
+  is split on whitespace characters to form a list of tokens. The first N1
+  tokens are used for the first column of the first row, where N1 is the first
+  element of the -colsize list. The next N2 are used for the second column of
+  the first row, and so on. Rows are added to the table until the entire list
+  of tokens is exhausted.
  }
 }

-if {$i > [llength $argv]-2} usage
-set O(db) [lindex $argv $i]
-set O(files) [lrange $argv [expr $i+1] end]
-
-foreach {k v} [lrange $argv 0 end-2] {
-  switch -- $k {
-    -colsize {
-      set O(aColSize) $v
+###########################################################################
+###########################################################################
+# Command line options processor. This is generic code that can be copied
+# between scripts.
+#
+namespace eval cmdline {
+  proc cmdline_error {O E {msg ""}} {
+    if {$msg != ""} {
+      puts stderr "Error: $msg"
+      puts stderr ""
    }
-
-    -colsize {
-      set O(aColSize) $v
+  
+    set L [list]
+    foreach o $O {
+      if {[llength $o]==1} {
+        lappend L [string toupper $o]
+      }
+    }
+  
+    puts stderr "Usage: $::argv0 ?SWITCHES? $L"
+    puts stderr ""
+    puts stderr "Switches are:"
+    foreach o $O {
+      if {[llength $o]==3} {
+        foreach {a b c} $o {}
+        puts stderr [format "    -%-15s %s (default \"%s\")" "$a VAL" $c $b]
+      } elseif {[llength $o]==2} {
+        foreach {a b} $o {}
+        puts stderr [format "    -%-15s %s" $a $b]
+      }
+    }
+    puts stderr ""
+    puts stderr $E
+    exit -1
+  }
+  
+  proc process {avar lArgs O E} {
+    upvar $avar A
+    set zTrailing ""       ;# True if ... is present in $O
+    set lPosargs [list]
+  
+    # Populate A() with default values. Also, for each switch in the command
+    # line spec, set an entry in the idx() array as follows:
+    #
+    #  {tblname t1 "table name to use"}  
+    #      -> [set idx(-tblname) {tblname t1 "table name to use"}  
+    #
+    # For each position parameter, append its name to $lPosargs. If the ...
+    # specifier is present, set $zTrailing to the name of the prefix.
+    #
+    foreach o $O {
+      set nm [lindex $o 0]
+      set nArg [llength $o]
+      switch -- $nArg {
+        1 {
+          if {[string range $nm end-2 end]=="..."} {
+            set zTrailing [string range $nm 0 end-3]
+          } else {
+            lappend lPosargs $nm
+          }
+        }
+        2 {
+          set A($nm) 0
+          set idx(-$nm) $o
+        }
+        3 {
+          set A($nm) [lindex $o 1]
+          set idx(-$nm) $o
+        }
+        default {
+          error "Error in command line specification"
+        }
+      }
+    }
+  
+    # Set explicitly specified option values
+    #
+    set nArg [llength $lArgs]
+    for {set i 0} {$i < $nArg} {incr i} {
+      set opt [lindex $lArgs $i]
+      if {[string range $opt 0 0]!="-" || $opt=="--"} break
+      set c [array names idx "${opt}*"]
+      if {[llength $c]==0} { cmdline_error $O $E "Unrecognized option: $opt"}
+      if {[llength $c]>1}  { cmdline_error $O $E "Ambiguous option: $opt"}
+  
+      if {[llength $idx($c)]==3} {
+        if {$i==[llength $lArgs]-1} {
+          cmdline_error $O $E "Option requires argument: $c" 
+        }
+        incr i
+        set A([lindex $idx($c) 0]) [lindex $lArgs $i]
+      } else {
+        set A([lindex $idx($c) 0]) 1
+      }
+    }
+  
+    # Deal with position arguments.
+    #
+    set nPosarg [llength $lPosargs]
+    set nRem [expr $nArg - $i]
+    if {$nRem < $nPosarg || ($zTrailing=="" && $nRem > $nPosarg)} {
+      cmdline_error $O $E
+    }
+    for {set j 0} {$j < $nPosarg} {incr j} {
+      set A([lindex $lPosargs $j]) [lindex $lArgs [expr $j+$i]]
+    }
+    if {$zTrailing!=""} {
+      set A($zTrailing) [lrange $lArgs [expr $j+$i] end]
    }
  }
+} ;# namespace eval cmdline
+# End of command line options processor.
+###########################################################################
+###########################################################################

+process_cmdline
+
+# If -fts4 was specified, use fts4. Otherwise, fts5.
+if {$A(fts4)} {
+  set A(fts) fts4
+} else {
+  set A(fts) fts5
 }

-sqlite3 db $O(db)
-load_static_extension db fts5
-
+sqlite3 db $A(database)

 # Create the FTS table in the db. Return a list of the table columns.
 #
 proc create_table {} {
-  global O
+  global A
  set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z]

-  set nCol [llength $O(aColsize)]
+  set nCol [llength $A(colsize)]
  set cols [lrange $cols 0 [expr $nCol-1]]

-  set sql    "CREATE VIRTUAL TABLE IF NOT EXISTS $O(tblname) USING $O(fts) ("
+  set sql    "CREATE VIRTUAL TABLE IF NOT EXISTS $A(tblname) USING $A(fts) ("
  append sql [join $cols ,]
-  append sql ");"
+  if {$A(fts)=="fts5"} { append sql ",detail=$A(detail)" }
+  append sql ", prefix='$A(prefix)');"

  db eval $sql
  return $cols
@@ -104,27 +189,35 @@ proc readfile {file} {
  split $data
 }

+proc repeat {L n} {
+  set res [list]
+  for {set i 0} {$i < $n} {incr i} {
+    set res [concat $res $L]
+  }
+  set res
+}
+

 # Load all the data into a big list of tokens.
 #
 set tokens [list]
-foreach f $O(files) {
-  set tokens [concat $tokens [readfile $f]]
+foreach f $A(file) {
+  set tokens [concat $tokens [repeat [readfile $f] $A(repeat)]]
 }

 set N [llength $tokens]
 set i 0
 set cols [create_table]
-set sql "INSERT INTO $O(tblname) VALUES(\$[lindex $cols 0]"
+set sql "INSERT INTO $A(tblname) VALUES(\$R([lindex $cols 0])"
 foreach c [lrange $cols 1 end] {
-  append sql ", \$$c"
+  append sql ", \$R($c)"
 }
 append sql ")"

 db eval BEGIN
  while {$i < $N} {
-    foreach c $cols s $O(aColsize) {
-      set $c [lrange $tokens $i [expr $i+$s-1]]
+    foreach c $cols s $A(colsize) {
+      set R($c) [lrange $tokens $i [expr $i+$s-1]]
      incr i $s
    }
    db eval $sql
@@ -48,6 +48,8 @@ proc usage {} {
  puts stderr "  -crisismerge N (set the crisismerge parameter to N)"
  puts stderr "  -prefix PREFIX (comma separated prefix= argument)"
  puts stderr "  -trans N     (commit after N inserts - 0 == never)"
+  puts stderr "  -hashsize N  (set the fts5 hashsize parameter to N)"
+  puts stderr "  -detail MODE (detail mode for fts5 tables)"
  exit 1
 }

@@ -59,6 +61,8 @@ set O(automerge)  -1
 set O(crisismerge)  -1
 set O(prefix)     ""
 set O(trans)      0
+set O(hashsize)   -1
+set O(detail)     full

 if {[llength $argv]<2} usage
 set nOpt [expr {[llength $argv]-2}]
@@ -106,6 +110,16 @@ for {set i 0} {$i < $nOpt} {incr i} {
      set O(prefix) [lindex $argv $i]
    }

+    -hashsize {
+      if { [incr i]>=$nOpt } usage
+      set O(hashsize) [lindex $argv $i]
+    }
+
+    -detail {
+      if { [incr i]>=$nOpt } usage
+      set O(detail) [lindex $argv $i]
+    }
+
    default {
      usage
    }
@@ -122,10 +136,21 @@ db eval "PRAGMA page_size=4096"
 db eval BEGIN
  set pref ""
  if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" }
+  if {$O(vtab)=="fts5"} {
+    append pref ", detail=$O(detail)"
+  }
  catch {
    db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok)$pref)"
    db eval "INSERT INTO t1(t1, rank) VALUES('pgsz', 4050);"
  }
+
+  if {$O(hashsize)>=0} {
+    catch {
+      db eval "INSERT INTO t1(t1, rank) VALUES('hashsize', $O(hashsize));"
+    }
+  }
+
+
  if {$O(automerge)>=0} {
    if {$O(vtab) == "fts5"} {
      db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) }
@@ -141,6 +166,7 @@ db eval BEGIN
  }
  load_hierachy [lindex $argv end]
 db eval COMMIT
+puts ""



@@ -78,7 +78,7 @@ proc fts5c_printfile {zIn} {
  global G
  set data [readfile $zIn]
  set zTail [file tail $zIn]
-  puts $G(fd) "#line 2 \"$zTail\""
+  puts $G(fd) "#line 1 \"$zTail\""

  set sub_map [list --FTS5-SOURCE-ID-- [fts5_source_id $::srcdir]]
  if {$zTail=="fts5parse.c"} {
@@ -86,8 +86,10 @@ proc fts5c_printfile {zIn} {
  }

  foreach line [split $data "\n"] {
-    if {[regexp {^#include.*fts5} $line]} continue
-    if { ![regexp { sqlite3Fts5Init\(} $line] 
+    if {[regexp {^#include.*fts5} $line]} {
+      set line "/* $line */"
+    } elseif { 
+         ![regexp { sqlite3Fts5Init\(} $line] 
       && [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?sqlite3Fts5} $line]
    } {
      set line "static $line"
@@ -8,11 +8,13 @@ proc usage {} {
  puts stderr "usage: $::argv0 ?OPTIONS? database table"
  puts stderr ""
  puts stderr "  -nterm                (count number of terms in each segment)"
+  puts stderr "  -segments             (output segment contents)"
  puts stderr ""
  exit 1
 }

 set O(nterm) 0
+set O(segments) 0

 if {[llength $argv]<2} usage
 foreach a [lrange $argv 0 end-2] {
@@ -21,6 +23,10 @@ foreach a [lrange $argv 0 end-2] {
      set O(nterm) 1
    }

+    -segments {
+      set O(segments) 1
+    }
+
    default {
      usage
    }
@@ -78,6 +84,13 @@ db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" {
  }
 }

+if {$O(segments)} {
+  puts ""
+  db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id>10" {
+    puts $d
+  }
+}
+



@@ -816,10 +816,10 @@ static const char *amatchValueOfKey(const char *zKey, const char *zStr){
  int i;
  if( nStr<nKey+1 ) return 0;
  if( memcmp(zStr, zKey, nKey)!=0 ) return 0;
-  for(i=nKey; isspace(zStr[i]); i++){}
+  for(i=nKey; isspace((unsigned char)zStr[i]); i++){}
  if( zStr[i]!='=' ) return 0;
  i++;
-  while( isspace(zStr[i]) ){ i++; }
+  while( isspace((unsigned char)zStr[i]) ){ i++; }
  return zStr+i;
 }

@@ -486,10 +486,10 @@ static const char *closureValueOfKey(const char *zKey, const char *zStr){
  int i;
  if( nStr<nKey+1 ) return 0;
  if( memcmp(zStr, zKey, nKey)!=0 ) return 0;
-  for(i=nKey; isspace(zStr[i]); i++){}
+  for(i=nKey; isspace((unsigned char)zStr[i]); i++){}
  if( zStr[i]!='=' ) return 0;
  i++;
-  while( isspace(zStr[i]) ){ i++; }
+  while( isspace((unsigned char)zStr[i]) ){ i++; }
  return zStr+i;
 }

@@ -94,16 +94,16 @@ static void ieee754func(
      m >>= 1;
      e++;
    }
-    while( ((m>>32)&0xfff00000)==0 ){
+    while( m!=0 && ((m>>32)&0xfff00000)==0 ){
      m <<= 1;
      e--;
    }
    e += 1075;
    if( e<0 ) e = m = 0;
-    if( e>0x7ff ) m = 0;
+    if( e>0x7ff ) e = 0x7ff;
    a = m & ((((sqlite3_int64)1)<<52)-1);
    a |= e<<52;
-    if( isNeg ) a |= ((sqlite3_int64)1)<<63;
+    if( isNeg ) a |= ((sqlite3_uint64)1)<<63;
    memcpy(&r, &a, sizeof(r));
    sqlite3_result_double(context, r);
  }
@@ -28,7 +28,6 @@
 SQLITE_EXTENSION_INIT1
 #include <assert.h>
 #include <string.h>
-#include <ctype.h>  /* amalgamator: keep */
 #include <stdlib.h>
 #include <stdarg.h>

@@ -43,8 +42,17 @@ SQLITE_EXTENSION_INIT1
 ** Versions of isspace(), isalnum() and isdigit() to which it is safe
 ** to pass signed char values.
 */
-#define safe_isdigit(x) isdigit((unsigned char)(x))
-#define safe_isalnum(x) isalnum((unsigned char)(x))
+#ifdef sqlite3Isdigit
+   /* Use the SQLite core versions if this routine is part of the
+   ** SQLite amalgamation */
+#  define safe_isdigit(x) sqlite3Isdigit(x)
+#  define safe_isalnum(x) sqlite3Isalnum(x)
+#else
+   /* Use the standard library for separate compilation */
+#include <ctype.h>  /* amalgamator: keep */
+#  define safe_isdigit(x) isdigit((unsigned char)(x))
+#  define safe_isalnum(x) isalnum((unsigned char)(x))
+#endif

 /*
 ** Growing our own isspace() routine this way is twice as fast as
@@ -52,7 +60,7 @@ SQLITE_EXTENSION_INIT1
 ** increase for the parser.  (Ubuntu14.10 gcc 4.8.4 x64 with -Os).
 */
 static const char jsonIsSpace[] = {
-  0, 0, 0, 0, 0, 0, 0, 0,     0, 1, 1, 0, 1, 1, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0,     0, 1, 1, 0, 0, 1, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
  1, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
@@ -268,10 +276,33 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){
  if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return;
  p->zBuf[p->nUsed++] = '"';
  for(i=0; i<N; i++){
-    char c = zIn[i];
+    unsigned char c = ((unsigned const char*)zIn)[i];
    if( c=='"' || c=='\\' ){
+      json_simple_escape:
      if( (p->nUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return;
      p->zBuf[p->nUsed++] = '\\';
+    }else if( c<=0x1f ){
+      static const char aSpecial[] = {
+         0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0,
+         0, 0, 0, 0, 0, 0, 0, 0,   0,   0,   0, 0,   0,   0, 0, 0
+      };
+      assert( sizeof(aSpecial)==32 );
+      assert( aSpecial['\b']=='b' );
+      assert( aSpecial['\f']=='f' );
+      assert( aSpecial['\n']=='n' );
+      assert( aSpecial['\r']=='r' );
+      assert( aSpecial['\t']=='t' );
+      if( aSpecial[c] ){
+        c = aSpecial[c];
+        goto json_simple_escape;
+      }
+      if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return;
+      p->zBuf[p->nUsed++] = '\\';
+      p->zBuf[p->nUsed++] = 'u';
+      p->zBuf[p->nUsed++] = '0';
+      p->zBuf[p->nUsed++] = '0';
+      p->zBuf[p->nUsed++] = '0' + (c>>4);
+      c = "0123456789abcdef"[c&0xf];
    }
    p->zBuf[p->nUsed++] = c;
  }
@@ -312,7 +343,7 @@ static void jsonAppendValue(
    default: {
      if( p->bErr==0 ){
        sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1);
-        p->bErr = 1;
+        p->bErr = 2;
        jsonReset(p);
      }
      break;
@@ -514,7 +545,13 @@ static void jsonReturn(
      int_as_real: /* fall through to real */;
    }
    case JSON_REAL: {
-      double r = strtod(pNode->u.zJContent, 0);
+      double r;
+#ifdef SQLITE_AMALGAMATION
+      const char *z = pNode->u.zJContent;
+      sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8);
+#else
+      r = strtod(pNode->u.zJContent, 0);
+#endif
      sqlite3_result_double(pCtx, r);
      break;
    }
@@ -1167,7 +1204,7 @@ static void jsonTest1Func(
 #endif /* SQLITE_DEBUG */

 /****************************************************************************
-** SQL function implementations
+** Scalar SQL function implementations
 ****************************************************************************/

 /*
@@ -1500,6 +1537,102 @@ static void jsonValidFunc(
  sqlite3_result_int(ctx, rc);
 }

+
+/****************************************************************************
+** Aggregate SQL function implementations
+****************************************************************************/
+/*
+** json_group_array(VALUE)
+**
+** Return a JSON array composed of all values in the aggregate.
+*/
+static void jsonArrayStep(
+  sqlite3_context *ctx,
+  int argc,
+  sqlite3_value **argv
+){
+  JsonString *pStr;
+  pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr));
+  if( pStr ){
+    if( pStr->zBuf==0 ){
+      jsonInit(pStr, ctx);
+      jsonAppendChar(pStr, '[');
+    }else{
+      jsonAppendChar(pStr, ',');
+      pStr->pCtx = ctx;
+    }
+    jsonAppendValue(pStr, argv[0]);
+  }
+}
+static void jsonArrayFinal(sqlite3_context *ctx){
+  JsonString *pStr;
+  pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
+  if( pStr ){
+    pStr->pCtx = ctx;
+    jsonAppendChar(pStr, ']');
+    if( pStr->bErr ){
+      if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
+      assert( pStr->bStatic );
+    }else{
+      sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
+                          pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
+      pStr->bStatic = 1;
+    }
+  }else{
+    sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC);
+  }
+  sqlite3_result_subtype(ctx, JSON_SUBTYPE);
+}
+
+/*
+** json_group_obj(NAME,VALUE)
+**
+** Return a JSON object composed of all names and values in the aggregate.
+*/
+static void jsonObjectStep(
+  sqlite3_context *ctx,
+  int argc,
+  sqlite3_value **argv
+){
+  JsonString *pStr;
+  const char *z;
+  u32 n;
+  pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr));
+  if( pStr ){
+    if( pStr->zBuf==0 ){
+      jsonInit(pStr, ctx);
+      jsonAppendChar(pStr, '{');
+    }else{
+      jsonAppendChar(pStr, ',');
+      pStr->pCtx = ctx;
+    }
+    z = (const char*)sqlite3_value_text(argv[0]);
+    n = (u32)sqlite3_value_bytes(argv[0]);
+    jsonAppendString(pStr, z, n);
+    jsonAppendChar(pStr, ':');
+    jsonAppendValue(pStr, argv[1]);
+  }
+}
+static void jsonObjectFinal(sqlite3_context *ctx){
+  JsonString *pStr;
+  pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
+  if( pStr ){
+    jsonAppendChar(pStr, '}');
+    if( pStr->bErr ){
+      if( pStr->bErr==0 ) sqlite3_result_error_nomem(ctx);
+      assert( pStr->bStatic );
+    }else{
+      sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
+                          pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
+      pStr->bStatic = 1;
+    }
+  }else{
+    sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC);
+  }
+  sqlite3_result_subtype(ctx, JSON_SUBTYPE);
+}
+
+
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 /****************************************************************************
 ** The json_each virtual table
@@ -1998,6 +2131,15 @@ int sqlite3Json1Init(sqlite3 *db){
    { "json_test1",           1, 0,   jsonTest1Func         },
 #endif
  };
+  static const struct {
+     const char *zName;
+     int nArg;
+     void (*xStep)(sqlite3_context*,int,sqlite3_value**);
+     void (*xFinal)(sqlite3_context*);
+  } aAgg[] = {
+    { "json_group_array",     1,   jsonArrayStep,   jsonArrayFinal  },
+    { "json_group_object",    2,   jsonObjectStep,  jsonObjectFinal },
+  };
 #ifndef SQLITE_OMIT_VIRTUALTABLE
  static const struct {
     const char *zName;
@@ -2013,6 +2155,11 @@ int sqlite3Json1Init(sqlite3 *db){
                                 (void*)&aFunc[i].flag,
                                 aFunc[i].xFunc, 0, 0);
  }
+  for(i=0; i<sizeof(aAgg)/sizeof(aAgg[0]) && rc==SQLITE_OK; i++){
+    rc = sqlite3_create_function(db, aAgg[i].zName, aAgg[i].nArg,
+                                 SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0,
+                                 0, aAgg[i].xStep, aAgg[i].xFinal);
+  }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
  for(i=0; i<sizeof(aMod)/sizeof(aMod[0]) && rc==SQLITE_OK; i++){
    rc = sqlite3_create_module(db, aMod[i].zName, aMod[i].pModule, 0);
@@ -2022,6 +2169,7 @@ int sqlite3Json1Init(sqlite3 *db){
 }


+#ifndef SQLITE_CORE
 #ifdef _WIN32
 __declspec(dllexport)
 #endif
@@ -2034,4 +2182,5 @@ int sqlite3_json_init(
  (void)pzErrMsg;  /* Unused parameter */
  return sqlite3Json1Init(db);
 }
+#endif
 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) */
@@ -186,7 +186,7 @@ static const unsigned char className[] = ".ABCDHLRMY9 ?";
 ** Return NULL if memory allocation fails.  
 */
 static unsigned char *phoneticHash(const unsigned char *zIn, int nIn){
-  unsigned char *zOut = sqlite3_malloc( nIn + 1 );
+  unsigned char *zOut = sqlite3_malloc64( nIn + 1 );
  int i;
  int nOut = 0;
  char cPrev = 0x77;
@@ -365,8 +365,8 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){
  int *m;                /* The cost matrix */
  char *cx;              /* Corresponding character values */
  int *toFree = 0;       /* Malloced space */
-  int mStack[60+15];     /* Stack space to use if not too much is needed */
  int nMatch = 0;
+  int mStack[60+15];     /* Stack space to use if not too much is needed */

  /* Early out if either input is NULL */
  if( zA==0 || zB==0 ) return -1;
@@ -413,7 +413,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){
  if( nB<(sizeof(mStack)*4)/(sizeof(mStack[0])*5) ){
    m = mStack;
  }else{
-    m = toFree = sqlite3_malloc( (nB+1)*5*sizeof(m[0])/4 );
+    m = toFree = sqlite3_malloc64( (nB+1)*5*sizeof(m[0])/4 );
    if( m==0 ) return -3;
  }
  cx = (char*)&m[nB+1];
@@ -687,7 +687,7 @@ static int editDist3ConfigLoad(
    if( iCost<0 ) continue;
    if( pLang==0 || iLang!=iLangPrev ){
      EditDist3Lang *pNew;
-      pNew = sqlite3_realloc(p->a, (p->nLang+1)*sizeof(p->a[0]));
+      pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0]));
      if( pNew==0 ){ rc = SQLITE_NOMEM; break; }
      p->a = pNew;
      pLang = &p->a[p->nLang];
@@ -709,7 +709,7 @@ static int editDist3ConfigLoad(
      EditDist3Cost *pCost;
      int nExtra = nFrom + nTo - 4;
      if( nExtra<0 ) nExtra = 0;
-      pCost = sqlite3_malloc( sizeof(*pCost) + nExtra );
+      pCost = sqlite3_malloc64( sizeof(*pCost) + nExtra );
      if( pCost==0 ){ rc = SQLITE_NOMEM; break; }
      pCost->nFrom = nFrom;
      pCost->nTo = nTo;
@@ -808,7 +808,7 @@ static EditDist3FromString *editDist3FromStringNew(

  if( z==0 ) return 0;
  if( n<0 ) n = (int)strlen(z);
-  pStr = sqlite3_malloc( sizeof(*pStr) + sizeof(pStr->a[0])*n + n + 1 );
+  pStr = sqlite3_malloc64( sizeof(*pStr) + sizeof(pStr->a[0])*n + n + 1 );
  if( pStr==0 ) return 0;
  pStr->a = (EditDist3From*)&pStr[1];
  memset(pStr->a, 0, sizeof(pStr->a[0])*n);
@@ -833,13 +833,13 @@ static EditDist3FromString *editDist3FromStringNew(
      if( i+p->nFrom>n ) continue;
      if( matchFrom(p, z+i, n-i)==0 ) continue;
      if( p->nTo==0 ){
-        apNew = sqlite3_realloc(pFrom->apDel,
+        apNew = sqlite3_realloc64(pFrom->apDel,
                                sizeof(*apNew)*(pFrom->nDel+1));
        if( apNew==0 ) break;
        pFrom->apDel = apNew;
        apNew[pFrom->nDel++] = p;
      }else{
-        apNew = sqlite3_realloc(pFrom->apSubst,
+        apNew = sqlite3_realloc64(pFrom->apSubst,
                                sizeof(*apNew)*(pFrom->nSubst+1));
        if( apNew==0 ) break;
        pFrom->apSubst = apNew;
@@ -875,6 +875,17 @@ static void updateCost(
  }
 }

+/*
+** How much stack space (int bytes) to use for Wagner matrix in 
+** editDist3Core().  If more space than this is required, the entire
+** matrix is taken from the heap.  To reduce the load on the memory
+** allocator, make this value as large as practical for the
+** architecture in use.
+*/
+#ifndef SQLITE_SPELLFIX_STACKALLOC_SZ
+# define SQLITE_SPELLFIX_STACKALLOC_SZ  (1024)
+#endif
+
 /* Compute the edit distance between two strings.
 **
 ** If an error occurs, return a negative number which is the error code.
@@ -899,15 +910,24 @@ static int editDist3Core(
  EditDist3FromString f = *pFrom;
  EditDist3To *a2;
  unsigned int *m;
+  unsigned int *pToFree;
  int szRow;
  EditDist3Cost *p;
  int res;
+  sqlite3_uint64 nByte;
+  unsigned int stackSpace[SQLITE_SPELLFIX_STACKALLOC_SZ/sizeof(unsigned int)];

  /* allocate the Wagner matrix and the aTo[] array for the TO string */
  n = (f.n+1)*(n2+1);
  n = (n+1)&~1;
-  m = sqlite3_malloc( n*sizeof(m[0]) + sizeof(a2[0])*n2 );
-  if( m==0 ) return -1;            /* Out of memory */
+  nByte = n*sizeof(m[0]) + sizeof(a2[0])*n2;
+  if( nByte<=sizeof(stackSpace) ){
+    m = stackSpace;
+    pToFree = 0;
+  }else{
+    m = pToFree = sqlite3_malloc64( nByte );
+    if( m==0 ) return -1;            /* Out of memory */
+  }
  a2 = (EditDist3To*)&m[n];
  memset(a2, 0, sizeof(a2[0])*n2);

@@ -920,7 +940,7 @@ static int editDist3Core(
      if( i2+p->nTo>n2 ) continue;
      if( matchTo(p, z2+i2, n2-i2)==0 ) continue;
      a2[i2].nIns++;
-      apNew = sqlite3_realloc(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns);
+      apNew = sqlite3_realloc64(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns);
      if( apNew==0 ){
        res = -1;  /* Out of memory */
        goto editDist3Abort;
@@ -1029,7 +1049,7 @@ static int editDist3Core(

 editDist3Abort:
  for(i2=0; i2<n2; i2++) sqlite3_free(a2[i2].apIns);
-  sqlite3_free(m);
+  sqlite3_free(pToFree);
  return res;
 }

@@ -1098,7 +1118,7 @@ static void editDist3SqlFunc(
 */
 static int editDist3Install(sqlite3 *db){
  int rc;
-  EditDist3Config *pConfig = sqlite3_malloc( sizeof(*pConfig) );
+  EditDist3Config *pConfig = sqlite3_malloc64( sizeof(*pConfig) );
  if( pConfig==0 ) return SQLITE_NOMEM;
  memset(pConfig, 0, sizeof(*pConfig));
  rc = sqlite3_create_function_v2(db, "editdist3",
@@ -1587,7 +1607,7 @@ static const struct {
 ** should be freed by the caller.
 */
 static unsigned char *transliterate(const unsigned char *zIn, int nIn){
-  unsigned char *zOut = sqlite3_malloc( nIn*4 + 1 );
+  unsigned char *zOut = sqlite3_malloc64( nIn*4 + 1 );
  int c, sz, nOut;
  if( zOut==0 ) return 0;
  nOut = 0;
@@ -1717,17 +1737,23 @@ static void scriptCodeSqlFunc(
 # define SCRIPT_LATIN       0x0001
 # define SCRIPT_CYRILLIC    0x0002
 # define SCRIPT_GREEK       0x0004
+# define SCRIPT_HEBREW      0x0008
+# define SCRIPT_ARABIC      0x0010

  while( nIn>0 ){
    c = utf8Read(zIn, nIn, &sz);
    zIn += sz;
    nIn -= sz;
-    if( c<0x02af ){
+    if( c<0x02af && (c>=0x80 || midClass[c&0x7f]<CCLASS_DIGIT) ){
      scriptMask |= SCRIPT_LATIN;
    }else if( c>=0x0400 && c<=0x04ff ){
      scriptMask |= SCRIPT_CYRILLIC;
    }else if( c>=0x0386 && c<=0x03ce ){
      scriptMask |= SCRIPT_GREEK;
+    }else if( c>=0x0590 && c<=0x05ff ){
+      scriptMask |= SCRIPT_HEBREW;
+    }else if( c>=0x0600 && c<=0x06ff ){
+      scriptMask |= SCRIPT_ARABIC;
    }
  }
  switch( scriptMask ){
@@ -1735,6 +1761,8 @@ static void scriptCodeSqlFunc(
    case SCRIPT_LATIN:     res = 215; break;
    case SCRIPT_CYRILLIC:  res = 220; break;
    case SCRIPT_GREEK:     res = 200; break;
+    case SCRIPT_HEBREW:    res = 125; break;
+    case SCRIPT_ARABIC:    res = 160; break;
    default:               res = 998; break;
  }
  sqlite3_result_int(context, res);
@@ -1851,7 +1879,7 @@ static char *spellfix1Dequote(const char *zIn){
  char *zOut;
  int i, j;
  char c;
-  while( isspace(zIn[0]) ) zIn++;
+  while( isspace((unsigned char)zIn[0]) ) zIn++;
  zOut = sqlite3_mprintf("%s", zIn);
  if( zOut==0 ) return 0;
  i = (int)strlen(zOut);
@@ -1902,7 +1930,7 @@ static int spellfix1Init(
  int i;

  nDbName = (int)strlen(zDbName);
-  pNew = sqlite3_malloc( sizeof(*pNew) + nDbName + 1);
+  pNew = sqlite3_malloc64( sizeof(*pNew) + nDbName + 1);
  if( pNew==0 ){
    rc = SQLITE_NOMEM;
  }else{
@@ -2016,7 +2044,7 @@ static void spellfix1ResetCursor(spellfix1_cursor *pCur){
 static void spellfix1ResizeCursor(spellfix1_cursor *pCur, int N){
  struct spellfix1_row *aNew;
  assert( N>=pCur->nRow );
-  aNew = sqlite3_realloc(pCur->a, sizeof(pCur->a[0])*N);
+  aNew = sqlite3_realloc64(pCur->a, sizeof(pCur->a[0])*N);
  if( aNew==0 && N>0 ){
    spellfix1ResetCursor(pCur);
    sqlite3_free(pCur->a);
@@ -2175,7 +2203,7 @@ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
 static int spellfix1Open(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
  spellfix1_vtab *p = (spellfix1_vtab*)pVTab;
  spellfix1_cursor *pCur;
-  pCur = sqlite3_malloc( sizeof(*pCur) );
+  pCur = sqlite3_malloc64( sizeof(*pCur) );
  if( pCur==0 ) return SQLITE_NOMEM;
  memset(pCur, 0, sizeof(*pCur));
  pCur->pVTab = p;
@@ -2389,7 +2417,7 @@ static int spellfix1FilterForMatch(

  /* Load the cost table if we have not already done so */
  if( p->zCostTable!=0 && p->pConfig3==0 ){
-    p->pConfig3 = sqlite3_malloc( sizeof(p->pConfig3[0]) );
+    p->pConfig3 = sqlite3_malloc64( sizeof(p->pConfig3[0]) );
    if( p->pConfig3==0 ) return SQLITE_NOMEM;
    memset(p->pConfig3, 0, sizeof(p->pConfig3[0]));
    rc = editDist3ConfigLoad(p->pConfig3, p->db, p->zCostTable);
@@ -75,7 +75,7 @@ int main(int argc, char **argv){
  /* Process command line arguments. Following this block local variables 
  ** zTarget, zRbu and nStep are all set. */
  if( argc==5 ){
-    int nArg1 = strlen(argv[1]);
+    size_t nArg1 = strlen(argv[1]);
    if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]);
    nStep = atoi(argv[2]);
  }else if( argc!=3 ){
@@ -103,7 +103,7 @@ int main(int argc, char **argv){
          "SQLITE_OK: rbu update incomplete (%lld operations so far)\n",
          nProgress
      );
-      fprintf(stdout, zBuf);
+      fprintf(stdout, "%s", zBuf);
      break;

    case SQLITE_DONE:
@@ -111,7 +111,7 @@ int main(int argc, char **argv){
          "SQLITE_DONE: rbu update completed (%lld operations)\n",
          nProgress
      );
-      fprintf(stdout, zBuf);
+      fprintf(stdout, "%s", zBuf);
      break;

    default:
@@ -122,4 +122,3 @@ int main(int argc, char **argv){
  sqlite3_free(zErrmsg);
  return (rc==SQLITE_OK || rc==SQLITE_DONE) ? 0 : 1;
 }
-
@@ -935,7 +935,7 @@ static void *rbuMalloc(sqlite3rbu *p, int nByte){
  void *pRet = 0;
  if( p->rc==SQLITE_OK ){
    assert( nByte>0 );
-    pRet = sqlite3_malloc(nByte);
+    pRet = sqlite3_malloc64(nByte);
    if( pRet==0 ){
      p->rc = SQLITE_NOMEM;
    }else{
@@ -981,8 +981,8 @@ static char *rbuStrndup(const char *zStr, int *pRc){

  assert( *pRc==SQLITE_OK );
  if( zStr ){
-    int nCopy = strlen(zStr) + 1;
-    zRet = (char*)sqlite3_malloc(nCopy);
+    size_t nCopy = strlen(zStr) + 1;
+    zRet = (char*)sqlite3_malloc64(nCopy);
    if( zRet ){
      memcpy(zRet, zStr, nCopy);
    }else{
@@ -2330,7 +2330,7 @@ static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){
  if( pRbu->nFrame==pRbu->nFrameAlloc ){
    int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2;
    RbuFrame *aNew;
-    aNew = (RbuFrame*)sqlite3_realloc(pRbu->aFrame, nNew * sizeof(RbuFrame));
+    aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame));
    if( aNew==0 ) return SQLITE_NOMEM;
    pRbu->aFrame = aNew;
    pRbu->nFrameAlloc = nNew;
@@ -2395,7 +2395,7 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){
  if( nChar==0 ){
    return 0;
  }
-  zWideFilename = sqlite3_malloc( nChar*sizeof(zWideFilename[0]) );
+  zWideFilename = sqlite3_malloc64( nChar*sizeof(zWideFilename[0]) );
  if( zWideFilename==0 ){
    return 0;
  }
@@ -3029,11 +3029,12 @@ sqlite3rbu *sqlite3rbu_open(
  const char *zState
 ){
  sqlite3rbu *p;
-  int nTarget = strlen(zTarget);
-  int nRbu = strlen(zRbu);
-  int nState = zState ? strlen(zState) : 0;
+  size_t nTarget = strlen(zTarget);
+  size_t nRbu = strlen(zRbu);
+  size_t nState = zState ? strlen(zState) : 0;
+  size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1;

-  p = (sqlite3rbu*)sqlite3_malloc(sizeof(sqlite3rbu)+nTarget+1+nRbu+1+nState+1);
+  p = (sqlite3rbu*)sqlite3_malloc64(nByte);
  if( p ){
    RbuState *pState = 0;

@@ -3170,7 +3171,7 @@ sqlite3 *sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){
 static void rbuEditErrmsg(sqlite3rbu *p){
  if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){
    int i;
-    int nErrmsg = strlen(p->zErrmsg);
+    size_t nErrmsg = strlen(p->zErrmsg);
    for(i=0; i<(nErrmsg-8); i++){
      if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){
        int nDel = 8;
@@ -3634,7 +3635,7 @@ static int rbuVfsShmMap(
  if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){
    if( iRegion<=p->nShm ){
      int nByte = (iRegion+1) * sizeof(char*);
-      char **apNew = (char**)sqlite3_realloc(p->apShm, nByte);
+      char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte);
      if( apNew==0 ){
        rc = SQLITE_NOMEM;
      }else{
@@ -3645,7 +3646,7 @@ static int rbuVfsShmMap(
    }

    if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){
-      char *pNew = (char*)sqlite3_malloc(szRegion);
+      char *pNew = (char*)sqlite3_malloc64(szRegion);
      if( pNew==0 ){
        rc = SQLITE_NOMEM;
      }else{
@@ -3755,7 +3756,7 @@ static int rbuVfsOpen(
      ** the name of the *-wal file this db connection will use. SQLite
      ** happens to pass a pointer to this buffer when using xAccess()
      ** or xOpen() to operate on the *-wal file.  */
-      int n = strlen(zName);
+      int n = (int)strlen(zName);
      const char *z = &zName[n];
      if( flags & SQLITE_OPEN_URI ){
        int odd = 0;
@@ -3781,8 +3782,8 @@ static int rbuVfsOpen(
          ** code ensures that the string passed to xOpen() is terminated by a
          ** pair of '\0' bytes in case the VFS attempts to extract a URI 
          ** parameter from it.  */
-          int nCopy = strlen(zName);
-          char *zCopy = sqlite3_malloc(nCopy+2);
+          size_t nCopy = strlen(zName);
+          char *zCopy = sqlite3_malloc64(nCopy+2);
          if( zCopy ){
            memcpy(zCopy, zName, nCopy);
            zCopy[nCopy-3] = 'o';
@@ -4011,13 +4012,13 @@ int sqlite3rbu_create_vfs(const char *zName, const char *zParent){
  };

  rbu_vfs *pNew = 0;              /* Newly allocated VFS */
-  int nName;
  int rc = SQLITE_OK;
+  size_t nName;
+  size_t nByte;

-  int nByte;
  nName = strlen(zName);
  nByte = sizeof(rbu_vfs) + nName + 1;
-  pNew = (rbu_vfs*)sqlite3_malloc(nByte);
+  pNew = (rbu_vfs*)sqlite3_malloc64(nByte);
  if( pNew==0 ){
    rc = SQLITE_NOMEM;
  }else{
@@ -341,6 +341,9 @@ sqlite3rbu *sqlite3rbu_open(
 ** If an error has occurred, either while opening or stepping the RBU object,
 ** this function may return NULL. The error code and message may be collected
 ** when sqlite3rbu_close() is called.
+**
+** Database handles returned by this function remain valid until the next
+** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db().
 */
 sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu);

@@ -56,20 +56,34 @@ static int test_sqlite3rbu_cmd(
 ){
  int ret = TCL_OK;
  sqlite3rbu *pRbu = (sqlite3rbu*)clientData;
-  const char *azMethod[] = { 
-    "step", "close", "create_rbu_delta", "savestate", 0 
+  struct RbuCmd {
+    const char *zName;
+    int nArg;
+    const char *zUsage;
+  } aCmd[] = {
+    {"step", 2, ""},              /* 0 */
+    {"close", 2, ""},             /* 1 */
+    {"create_rbu_delta", 2, ""},  /* 2 */
+    {"savestate", 2, ""},         /* 3 */
+    {"dbMain_eval", 3, "SQL"},    /* 4 */
+    {0,0,0}
  };
-  int iMethod;
+  int iCmd;

-  if( objc!=2 ){
+  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "METHOD");
    return TCL_ERROR;
  }
-  if( Tcl_GetIndexFromObj(interp, objv[1], azMethod, "method", 0, &iMethod) ){
+  ret = Tcl_GetIndexFromObjStruct(
+      interp, objv[1], aCmd, sizeof(aCmd[0]), "method", 0, &iCmd
+  );
+  if( ret ) return TCL_ERROR;
+  if( objc!=aCmd[iCmd].nArg ){
+    Tcl_WrongNumArgs(interp, 1, objv, aCmd[iCmd].zUsage);
    return TCL_ERROR;
  }

-  switch( iMethod ){
+  switch( iCmd ){
    case 0: /* step */ {
      int rc = sqlite3rbu_step(pRbu);
      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
@@ -112,6 +126,16 @@ static int test_sqlite3rbu_cmd(
      break;
    }

+    case 4: /* dbMain_eval */ {
+      sqlite3 *db = sqlite3rbu_db(pRbu, 0);
+      int rc = sqlite3_exec(db, Tcl_GetString(objv[2]), 0, 0, 0);
+      if( rc!=SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errmsg(db), -1));
+        ret = TCL_ERROR;
+      }
+      break;
+    }
+
    default: /* seems unlikely */
      assert( !"cannot happen" );
      break;
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .9.0
 .11.0